maxframe 0.1.0b4__cp38-cp38-win32.whl → 1.0.0rc1__cp38-cp38-win32.whl

maxframe_client/session/task.py

@@ -21,12 +21,12 @@ from typing import Dict, List, Optional, Type, Union
 import msgpack
 from odps import ODPS
 from odps import options as odps_options
-from odps import serializers
 from odps.errors import parse_instance_error
-from odps.models import Instance, Task
+from odps.models import Instance, MaxFrameTask
 
 from maxframe.config import options
 from maxframe.core import TileableGraph
+from maxframe.errors import NoTaskServerResponseError, SessionAlreadyClosedError
 from maxframe.protocol import DagInfo, JsonSerializable, ResultInfo, SessionInfo
 from maxframe.utils import deserialize_serializable, serialize_serializable, to_str
 
@@ -55,55 +55,6 @@ from .odps import MaxFrameServiceCaller, MaxFrameSession
 logger = logging.getLogger(__name__)
 
 
-class MaxFrameTask(Task):
-    __slots__ = ("_output_format", "_major_version", "_service_endpoint")
-    _root = "MaxFrame"
-    _anonymous_task_name = "AnonymousMaxFrameTask"
-
-    command = serializers.XMLNodeField("Command", default="CREATE_SESSION")
-
-    def __init__(self, **kwargs):
-        kwargs["name"] = kwargs.get("name") or self._anonymous_task_name
-        self._output_format = kwargs.pop(
-            "output_format", MAXFRAME_OUTPUT_MSGPACK_FORMAT
-        )
-        self._major_version = kwargs.pop("major_version", None)
-        self._service_endpoint = kwargs.pop("service_endpoint", None)
-        super().__init__(**kwargs)
-
-    def serial(self):
-        if self.properties is None:
-            self.properties = dict()
-
-        if odps_options.default_task_settings:
-            settings = odps_options.default_task_settings
-        else:
-            settings = dict()
-
-        if self._major_version is not None:
-            settings["odps.task.major.version"] = self._major_version
-
-        if "settings" in self.properties:
-            settings.update(json.loads(self.properties["settings"]))
-
-        # merge sql options
-        sql_settings = (odps_options.sql.settings or {}).copy()
-        sql_settings.update(options.sql.settings or {})
-
-        mf_settings = dict(options.to_dict(remote_only=True).items())
-        mf_settings["sql.settings"] = sql_settings
-        mf_opts = {
-            "odps.maxframe.settings": json.dumps(mf_settings),
-            "odps.maxframe.output_format": self._output_format,
-            "odps.service.endpoint": self._service_endpoint,
-        }
-        if mf_version:
-            mf_opts["odps.maxframe.client_version"] = mf_version
-        settings.update(mf_opts)
-        self.properties["settings"] = json.dumps(settings)
-        return super().serial()
-
-
 class MaxFrameInstanceCaller(MaxFrameServiceCaller):
     _instance: Optional[Instance]
 
@@ -132,6 +83,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         self._running_cluster = running_cluster
         self._major_version = major_version
         self._output_format = output_format or MAXFRAME_OUTPUT_MSGPACK_FORMAT
+        self._deleted = False
 
         if nested_instance_id is None:
             self._nested = False
@@ -144,10 +96,18 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         self, content: Union[bytes, str, dict], target_cls: Type[JsonSerializable]
     ):
         if isinstance(content, (str, bytes)):
+            if len(content) == 0:
+                content = "{}"
             json_data = json.loads(to_str(content))
         else:
             json_data = content
-        result_data = base64.b64decode(json_data["result"])
+        encoded_result = json_data.get("result")
+        if not encoded_result:
+            if self._deleted:
+                return None
+            else:
+                raise SessionAlreadyClosedError(self._instance.id)
+        result_data = base64.b64decode(encoded_result)
         if self._output_format == MAXFRAME_OUTPUT_MAXFRAME_FORMAT:
             return deserialize_serializable(result_data)
         elif self._output_format == MAXFRAME_OUTPUT_JSON_FORMAT:
@@ -159,13 +119,31 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
                 f"Serialization format {self._output_format} not supported"
             )
 
-    def create_session(self) -> SessionInfo:
+    def _create_maxframe_task(self) -> MaxFrameTask:
         task = MaxFrameTask(
             name=self._task_name,
             major_version=self._major_version,
-            output_format=self._output_format,
             service_endpoint=self._odps_entry.endpoint,
         )
+
+        # merge sql options
+        sql_settings = (odps_options.sql.settings or {}).copy()
+        sql_settings.update(options.sql.settings or {})
+
+        mf_settings = dict(options.to_dict(remote_only=True).items())
+        mf_settings["sql.settings"] = sql_settings
+
+        mf_opts = {
+            "odps.maxframe.settings": json.dumps(mf_settings),
+            "odps.maxframe.output_format": self._output_format,
+        }
+        if mf_version:
+            mf_opts["odps.maxframe.client_version"] = mf_version
+        task.update_settings(mf_opts)
+        return task
+
+    def create_session(self) -> SessionInfo:
+        task = self._create_maxframe_task()
         if not self._nested:
             self._task_name = task.name
             project = self._odps_entry.get_project(self._project)
@@ -210,6 +188,14 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             time.sleep(interval)
             interval = min(max_interval, interval * 2)
 
+    def _put_task_info(self, method_name: str, json_data: dict):
+        resp_data = self._instance.put_task_info(
+            self._task_name, method_name, json.dumps(json_data)
+        )
+        if not resp_data:
+            raise NoTaskServerResponseError(f"No response for request {method_name}")
+        return resp_data
+
     def get_session(self) -> SessionInfo:
         req_data = {"output_format": self._output_format}
         serialized = self._instance.put_task_info(
@@ -224,11 +210,8 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             self._instance.stop()
         else:
             req_data = {"output_format": self._output_format}
-            self._instance.put_task_info(
-                self._task_name,
-                MAXFRAME_TASK_DELETE_SESSION_METHOD,
-                json.dumps(req_data),
-            )
+            self._put_task_info(MAXFRAME_TASK_DELETE_SESSION_METHOD, req_data)
+        self._deleted = True
 
     def submit_dag(
         self,
@@ -243,9 +226,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             ).decode(),
             "output_format": self._output_format,
         }
-        res = self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_SUBMIT_DAG_METHOD, json.dumps(req_data)
-        )
+        res = self._put_task_info(MAXFRAME_TASK_SUBMIT_DAG_METHOD, req_data)
         return self._deserial_task_info_result(res, DagInfo)
 
     def get_dag_info(self, dag_id: str) -> DagInfo:
@@ -254,9 +235,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             "dag_id": dag_id,
             "output_format": self._output_format,
         }
-        res = self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_GET_DAG_INFO_METHOD, json.dumps(req_data)
-        )
+        res = self._put_task_info(MAXFRAME_TASK_GET_DAG_INFO_METHOD, req_data)
         return self._deserial_task_info_result(res, DagInfo)
 
     def cancel_dag(self, dag_id: str) -> DagInfo:
@@ -265,18 +244,33 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             "dag_id": dag_id,
             "output_format": self._output_format,
         }
-        res = self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_CANCEL_DAG_METHOD, json.dumps(req_data)
-        )
+        res = self._put_task_info(MAXFRAME_TASK_CANCEL_DAG_METHOD, req_data)
         return self._deserial_task_info_result(res, DagInfo)
 
     def decref(self, tileable_keys: List[str]) -> None:
         req_data = {
             "tileable_keys": ",".join(tileable_keys),
         }
-        self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_DECREF_METHOD, json.dumps(req_data)
-        )
+        self._put_task_info(MAXFRAME_TASK_DECREF_METHOD, req_data)
+
+    def get_logview_address(self, dag_id=None, hours=None) -> Optional[str]:
+        """
+        Generate logview address
+
+        Parameters
+        ----------
+            dag_id: id of dag for which dag logview detail page to access
+            hours: hours of the logview address auth limit
+        Returns
+        -------
+            Logview address
+        """
+        hours = hours or options.session.logview_hours
+        # notice: maxframe can't reuse subQuery else will conflict with mcqa when fetch resource data,
+        #         added dagId for maxframe so logview backend will return maxframe data format if
+        #         instance and dagId is provided.
+        dag_suffix = f"&dagId={dag_id}" if dag_id else ""
+        return self._instance.get_logview_address(hours) + dag_suffix
 
 
 class MaxFrameTaskSession(MaxFrameSession):

maxframe_client/tests/test_session.py

@@ -23,7 +23,10 @@ from odps import ODPS
 
 import maxframe.dataframe as md
 import maxframe.remote as mr
+from maxframe.config import options
+from maxframe.config.config import option_context
 from maxframe.core import ExecutableTuple, TileableGraph
+from maxframe.errors import NoTaskServerResponseError
 from maxframe.lib.aio import stop_isolation
 from maxframe.protocol import ResultInfo
 from maxframe.serialization import RemoteException
@@ -35,6 +38,7 @@ from maxframe_framedriver.app.tests.test_framedriver_webapp import (  # noqa: F4
 )
 
 from ..clients.framedriver import FrameDriverClient
+from ..session.odps import MaxFrameRestCaller
 
 pytestmark = pytest.mark.maxframe_engine(["MCSQL", "SPE"])
 
@@ -86,11 +90,25 @@ def test_simple_run_dataframe(start_mock_session):
         assert len(dag) == 2
         return await original_submit_dag(self, session_id, dag, managed_input_infos)
 
+    no_task_server_raised = False
+    original_get_dag_info = MaxFrameRestCaller.get_dag_info
+
+    async def patched_get_dag_info(self, dag_id: str):
+        nonlocal no_task_server_raised
+
+        if not no_task_server_raised:
+            no_task_server_raised = True
+            raise NoTaskServerResponseError
+        return await original_get_dag_info(self, dag_id)
+
     df["H"] = "extra_content"
 
     with mock.patch(
         "maxframe_client.clients.framedriver.FrameDriverClient.submit_dag",
         new=patched_submit_dag,
+    ), mock.patch(
+        "maxframe_client.session.odps.MaxFrameRestCaller.get_dag_info",
+        new=patched_get_dag_info,
     ):
         result = df.execute().fetch()
         assert len(result) == 1000
@@ -99,9 +117,12 @@ def test_simple_run_dataframe(start_mock_session):
     corner_top, corner_bottom = ExecutableTuple([df.iloc[:10], df.iloc[-10:]]).fetch()
     assert len(corner_top) == len(corner_bottom) == 10
 
-    # check ellipsis mark in DataFrame errors
+    # check ellipsis mark in DataFrame reprs
     df_str_repr = str(df)
     assert ".." in df_str_repr
+    # check ellipsis mark in Series reprs
+    series_str_repr = str(df.A.execute())
+    assert ".." in series_str_repr
 
     key = df.key
     assert odps_entry.exist_table(
@@ -181,6 +202,24 @@ def test_run_dataframe_from_to_odps_table(start_mock_session):
         odps_entry.delete_table(table_name, if_exists=True)
 
 
+def test_create_session_with_options(framedriver_app):  # noqa: F811
+    odps_entry = ODPS.from_environments()
+    framedriver_addr = f"mf://localhost:{framedriver_app.port}"
+    old_value = options.session.max_alive_seconds
+    session = None
+    try:
+        options.session.max_alive_seconds = 10
+        session = new_session(framedriver_addr, odps_entry=odps_entry)
+        session_id = session.session_id
+        session_conf = framedriver_app.session_manager.get_session_settings(session_id)
+        with option_context(session_conf) as session_options:
+            assert session_options.session.max_alive_seconds == 10
+    finally:
+        options.session.max_alive_seconds = old_value
+        if session is not None:
+            session.destroy()
+
+
 def test_run_and_fetch_series(start_mock_session):
     odps_entry = ODPS.from_environments()
 
@@ -226,3 +265,27 @@ def test_run_remote_error(start_mock_session):
 
     with pytest.raises((ValueError, RemoteException)):
         v.execute()
+
+
+def test_pivot_dataframe(start_mock_session):
+    pd_df = pd.DataFrame(
+        {
+            "A": "foo foo foo foo foo bar bar bar bar".split(),
+            "B": "one one one two two one one two two".split(),
+            "C": "small large large small small large small small large".split(),
+            "D": [1, 2, 2, 3, 3, 4, 5, 6, 7],
+            "E": [2, 4, 5, 5, 6, 6, 8, 9, 9],
+        }
+    )
+    df = md.DataFrame(pd_df)
+    pivot = df.pivot_table(values="D", index=["A", "B"], columns=["C"], aggfunc="sum")
+    executed = pivot.execute()
+    assert pivot.shape == (2, 4)
+    pd.testing.assert_index_equal(
+        pivot.dtypes.index, pd.Index(["large", "small"], name="C")
+    )
+
+    expected = pd_df.pivot_table(
+        values="D", index=["A", "B"], columns=["C"], aggfunc="sum"
+    )
+    pd.testing.assert_frame_equal(executed.to_pandas(), expected)

maxframe_client/clients/spe.py

@@ -1,104 +0,0 @@
-# Copyright 1999-2024 Alibaba Group Holding Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any, Dict, Optional
-
-from tornado import httpclient
-
-from maxframe.core import TileableGraph
-from maxframe.protocol import ExecuteSubDagRequest, ProtocolBody, SubDagInfo
-from maxframe.typing_ import TimeoutType
-from maxframe.utils import (
-    deserialize_serializable,
-    format_timeout_params,
-    serialize_serializable,
-    wait_http_response,
-)
-
-
-class SPEClient:
-    def __init__(
-        self,
-        endpoint: str,
-        session_id: Optional[str] = None,
-        host: str = None,
-    ):
-        self._endpoint = endpoint.rstrip("/")
-        self._session_id = session_id
-        self._headers = {"Host": host}
-
-    @staticmethod
-    def _load_subdag_info(resp: httpclient.HTTPResponse) -> SubDagInfo:
-        res: ProtocolBody[SubDagInfo] = deserialize_serializable(resp.body)
-        return res.body
-
-    async def submit_subdag(
-        self, subdag: TileableGraph, settings: Dict[str, Any] = None
-    ) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags"
-        req_body: ProtocolBody[ExecuteSubDagRequest] = ProtocolBody(
-            body=ExecuteSubDagRequest(dag=subdag, settings=settings),
-        )
-
-        if self._session_id is not None:
-            req_url += f"?session_id={self._session_id}"
-
-        resp = await httpclient.AsyncHTTPClient().fetch(
-            req_url,
-            method="POST",
-            headers=self._headers,
-            body=serialize_serializable(req_body),
-        )
-        return self._load_subdag_info(resp)
-
-    async def get_subdag_info(self, subdag_id: str) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags/{subdag_id}?wait=0"
-        resp = await httpclient.AsyncHTTPClient().fetch(
-            req_url,
-            method="GET",
-            headers=self._headers,
-        )
-        return self._load_subdag_info(resp)
-
-    async def wait_subdag(
-        self, subdag_id: str, wait_timeout: TimeoutType = None
-    ) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags/{subdag_id}"
-        params = format_timeout_params(wait_timeout)
-        try:
-            resp = await wait_http_response(
-                req_url + params,
-                method="GET",
-                headers=self._headers,
-                request_timeout=wait_timeout,
-            )
-            return self._load_subdag_info(resp)
-        except TimeoutError:
-            return await self.get_subdag_info(subdag_id)
-
-    async def cancel_subdag(
-        self, subdag_id: str, wait_timeout: TimeoutType = None
-    ) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags/{subdag_id}"
-        params = format_timeout_params(wait_timeout)
-        try:
-            resp = await wait_http_response(
-                req_url + params,
-                method="DELETE",
-                headers=self._headers,
-                request_timeout=wait_timeout,
-            )
-            return self._load_subdag_info(resp)
-        except TimeoutError:
-            return await self.get_subdag_info(subdag_id)