maxframe 0.1.0b2__cp310-cp310-macosx_10_9_universal2.whl → 0.1.0b4__cp310-cp310-macosx_10_9_universal2.whl

maxframe_client/clients/framedriver.py

@@ -14,6 +14,7 @@
 
 from typing import Any, Dict, List
 
+import msgpack
 from tornado import httpclient
 
 from maxframe.core import TileableGraph
@@ -28,7 +29,6 @@ from maxframe.protocol import (
 )
 from maxframe.typing_ import TimeoutType
 from maxframe.utils import (
-    deserialize_serializable,
     format_timeout_params,
     serialize_serializable,
     wait_http_response,
@@ -47,12 +47,12 @@ class FrameDriverClient:
         resp = await httpclient.AsyncHTTPClient().fetch(
             req_url, method="POST", body=serialize_serializable(req_body)
         )
-        return deserialize_serializable(resp.body).body
+        return SessionInfo.from_json(msgpack.loads(resp.body))
 
     async def get_session(self, session_id: str) -> SessionInfo:
         req_url = f"{self._endpoint}/api/sessions/{session_id}"
         resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
-        return deserialize_serializable(resp.body).body
+        return SessionInfo.from_json(msgpack.loads(resp.body))
 
     async def delete_session(self, session_id: str):
         req_url = f"{self._endpoint}/api/sessions/{session_id}"
@@ -71,12 +71,12 @@ class FrameDriverClient:
             method="POST",
             body=serialize_serializable(ProtocolBody(body=req_body)),
         )
-        return deserialize_serializable(resp.body).body
+        return DagInfo.from_json(msgpack.loads(resp.body))
 
     async def get_dag_info(self, session_id: str, dag_id: str) -> DagInfo:
         req_url = f"{self._endpoint}/api/sessions/{session_id}/dags/{dag_id}"
         resp = await httpclient.AsyncHTTPClient().fetch(req_url, method="GET")
-        return deserialize_serializable(resp.body).body
+        return DagInfo.from_json(msgpack.loads(resp.body))
 
     async def wait_dag(self, session_id: str, dag_id: str, timeout: TimeoutType = None):
         query_part = format_timeout_params(timeout)
@@ -87,7 +87,7 @@ class FrameDriverClient:
             resp = await wait_http_response(
                 req_url, method="GET", request_timeout=timeout
             )
-            info = deserialize_serializable(resp.body).body
+            info = DagInfo.from_json(msgpack.loads(resp.body))
         except TimeoutError:
             info = await self.get_dag_info(session_id, dag_id)
         return info
@@ -103,7 +103,7 @@ class FrameDriverClient:
             resp = await wait_http_response(
                 req_url, method="DELETE", request_timeout=timeout
             )
-            info = deserialize_serializable(resp.body).body
+            info = DagInfo.from_json(msgpack.loads(resp.body))
         except TimeoutError:
             info = await self.get_dag_info(session_id, dag_id)
         return info

maxframe_client/session/odps.py

@@ -115,7 +115,7 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
     ):
         super().__init__(address, session_id)
         self.timeout = timeout
-        self._odps_entry = odps_entry or ODPS.from_environments()
+        self._odps_entry = odps_entry or ODPS.from_global() or ODPS.from_environments()
         self._tileable_to_infos = weakref.WeakKeyDictionary()
 
         self._caller = self._create_caller(odps_entry, address, **kwargs)
@@ -147,15 +147,16 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
         data = t.op.get_data()
         batch_size = options.session.upload_batch_size
 
-        halo_client = HaloTableIO(self._odps_entry)
-        with halo_client.open_writer(table_obj.full_table_name) as writer:
-            for batch_start in range(0, len(data), batch_size):
-                if isinstance(data, pd.Index):
-                    batch = data[batch_start : batch_start + batch_size]
-                else:
-                    batch = data.iloc[batch_start : batch_start + batch_size]
-                arrow_batch, _ = pandas_to_arrow(batch)
-                writer.write(arrow_batch)
+        if len(data):
+            halo_client = HaloTableIO(self._odps_entry)
+            with halo_client.open_writer(table_obj.full_table_name) as writer:
+                for batch_start in range(0, len(data), batch_size):
+                    if isinstance(data, pd.Index):
+                        batch = data[batch_start : batch_start + batch_size]
+                    else:
+                        batch = data.iloc[batch_start : batch_start + batch_size]
+                    arrow_batch, _ = pandas_to_arrow(batch)
+                    writer.write(arrow_batch)
 
         read_tileable = read_odps_table(
             table_obj.full_table_name,

maxframe_client/session/task.py

@@ -112,13 +112,19 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         odps_entry: ODPS,
         task_name: Optional[str] = None,
         project: Optional[str] = None,
-        priority: Optional[str] = None,
+        priority: Optional[int] = None,
         running_cluster: Optional[str] = None,
         nested_instance_id: Optional[str] = None,
         major_version: Optional[str] = None,
         output_format: Optional[str] = None,
         **kwargs,
     ):
+        if callable(odps_options.get_priority):
+            default_priority = odps_options.get_priority(odps_entry)
+        else:
+            default_priority = odps_options.priority
+        priority = priority if priority is not None else default_priority
+
         self._odps_entry = odps_entry
         self._task_name = task_name
         self._project = project
@@ -126,6 +132,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         self._running_cluster = running_cluster
         self._major_version = major_version
         self._output_format = output_format or MAXFRAME_OUTPUT_MSGPACK_FORMAT
+
         if nested_instance_id is None:
             self._nested = False
             self._instance = None

maxframe_client/session/tests/test_task.py

@@ -18,6 +18,7 @@ import os
 import mock
 from defusedxml import ElementTree
 from odps import ODPS
+from odps import options as odps_options
 
 from ...session.consts import MAXFRAME_OUTPUT_JSON_FORMAT
 from ...session.task import MaxFrameInstanceCaller, MaxFrameTask
@@ -27,17 +28,20 @@ expected_file_dir = os.path.join(os.path.dirname(__file__), "expected-data")
 
 def test_maxframe_instance_caller_creating_session():
     o = ODPS.from_environments()
-    task_caller = MaxFrameInstanceCaller(
-        odps_entry=o,
-        task_name="task_test",
-        major_version="test_version",
-        output_format=MAXFRAME_OUTPUT_JSON_FORMAT,
-        priority="100",
-        running_cluster="test_cluster",
-    )
+
+    def create_caller(**kwargs):
+        kw = dict(
+            odps_entry=o,
+            task_name="task_test",
+            major_version="test_version",
+            output_format=MAXFRAME_OUTPUT_JSON_FORMAT,
+            running_cluster="test_cluster",
+        )
+        kw.update(**kwargs)
+        return MaxFrameInstanceCaller(**kw)
 
     def mock_create(self, task: MaxFrameTask, priority=None, running_cluster=None):
-        assert priority == "100"
+        assert priority == 100
         assert running_cluster == "test_cluster"
         root = ElementTree.parse(
             os.path.join(expected_file_dir, "create_session.xml")
@@ -62,6 +66,20 @@ def test_maxframe_instance_caller_creating_session():
         target="maxframe_client.session.task.MaxFrameInstanceCaller",
         _wait_instance_task_ready=mock.DEFAULT,
         get_session=mock.DEFAULT,
-    ):
-        with mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+    ), mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+        task_caller = create_caller(priority=100)
+        task_caller.create_session()
+
+        old_priority = odps_options.priority
+        old_get_priority = odps_options.get_priority
+        try:
+            task_caller = create_caller(priority=100)
+            odps_options.priority = 100
+            task_caller.create_session()
+
+            odps_options.priority = None
+            odps_options.get_priority = lambda _: 100
             task_caller.create_session()
+        finally:
+            odps_options.priority = old_priority
+            odps_options.get_priority = old_get_priority

maxframe_client/tests/test_session.py

@@ -24,9 +24,11 @@ from odps import ODPS
 import maxframe.dataframe as md
 import maxframe.remote as mr
 from maxframe.core import ExecutableTuple, TileableGraph
+from maxframe.lib.aio import stop_isolation
 from maxframe.protocol import ResultInfo
 from maxframe.serialization import RemoteException
 from maxframe.session import new_session
+from maxframe.tests.utils import tn
 from maxframe.utils import build_temp_table_name
 from maxframe_framedriver.app.tests.test_framedriver_webapp import (  # noqa: F401
     framedriver_app,
@@ -52,6 +54,7 @@ def start_mock_session(framedriver_app):  # noqa: F811
             time.sleep(5)  # Wait for temp table deleted
         else:
             session.reset_default()
+        stop_isolation()
 
 
 def test_simple_run_dataframe(start_mock_session):
@@ -113,6 +116,26 @@ def test_simple_run_dataframe(start_mock_session):
     assert not odps_entry.exist_table(build_temp_table_name(start_mock_session, key))
 
 
+def test_run_empty_table(start_mock_session):
+    odps_entry = ODPS.from_environments()
+
+    table_name = tn("test_session_empty_table")
+    odps_entry.delete_table(table_name, if_exists=True)
+    empty_table = odps_entry.create_table(
+        table_name, "_idx_0 bigint, a double, b double", lifecycle=1
+    )
+    df = md.read_odps_table(table_name, index_col="_idx_0")
+    df["d"] = df["a"] + df["b"]
+
+    executed = df.execute()
+    assert "Index: []" in str(executed)
+
+    fetched = executed.fetch()
+    assert 0 == len(fetched)
+
+    empty_table.drop()
+
+
 def test_run_dataframe_with_pd_source(start_mock_session):
     odps_entry = ODPS.from_environments()