maxframe 0.1.0b5__cp310-cp310-macosx_10_9_universal2.whl → 1.0.0rc1__cp310-cp310-macosx_10_9_universal2.whl

maxframe_client/session/task.py

@@ -26,6 +26,7 @@ from odps.models import Instance, MaxFrameTask
 
 from maxframe.config import options
 from maxframe.core import TileableGraph
+from maxframe.errors import NoTaskServerResponseError, SessionAlreadyClosedError
 from maxframe.protocol import DagInfo, JsonSerializable, ResultInfo, SessionInfo
 from maxframe.utils import deserialize_serializable, serialize_serializable, to_str
 
@@ -82,6 +83,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         self._running_cluster = running_cluster
         self._major_version = major_version
         self._output_format = output_format or MAXFRAME_OUTPUT_MSGPACK_FORMAT
+        self._deleted = False
 
         if nested_instance_id is None:
             self._nested = False
@@ -94,10 +96,18 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
         self, content: Union[bytes, str, dict], target_cls: Type[JsonSerializable]
     ):
         if isinstance(content, (str, bytes)):
+            if len(content) == 0:
+                content = "{}"
             json_data = json.loads(to_str(content))
         else:
             json_data = content
-        result_data = base64.b64decode(json_data["result"])
+        encoded_result = json_data.get("result")
+        if not encoded_result:
+            if self._deleted:
+                return None
+            else:
+                raise SessionAlreadyClosedError(self._instance.id)
+        result_data = base64.b64decode(encoded_result)
         if self._output_format == MAXFRAME_OUTPUT_MAXFRAME_FORMAT:
             return deserialize_serializable(result_data)
         elif self._output_format == MAXFRAME_OUTPUT_JSON_FORMAT:
@@ -178,6 +188,14 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             time.sleep(interval)
             interval = min(max_interval, interval * 2)
 
+    def _put_task_info(self, method_name: str, json_data: dict):
+        resp_data = self._instance.put_task_info(
+            self._task_name, method_name, json.dumps(json_data)
+        )
+        if not resp_data:
+            raise NoTaskServerResponseError(f"No response for request {method_name}")
+        return resp_data
+
     def get_session(self) -> SessionInfo:
         req_data = {"output_format": self._output_format}
         serialized = self._instance.put_task_info(
@@ -192,11 +210,8 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             self._instance.stop()
         else:
             req_data = {"output_format": self._output_format}
-            self._instance.put_task_info(
-                self._task_name,
-                MAXFRAME_TASK_DELETE_SESSION_METHOD,
-                json.dumps(req_data),
-            )
+            self._put_task_info(MAXFRAME_TASK_DELETE_SESSION_METHOD, req_data)
+        self._deleted = True
 
     def submit_dag(
         self,
@@ -211,9 +226,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             ).decode(),
             "output_format": self._output_format,
         }
-        res = self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_SUBMIT_DAG_METHOD, json.dumps(req_data)
-        )
+        res = self._put_task_info(MAXFRAME_TASK_SUBMIT_DAG_METHOD, req_data)
         return self._deserial_task_info_result(res, DagInfo)
 
     def get_dag_info(self, dag_id: str) -> DagInfo:
@@ -222,9 +235,7 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             "dag_id": dag_id,
             "output_format": self._output_format,
         }
-        res = self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_GET_DAG_INFO_METHOD, json.dumps(req_data)
-        )
+        res = self._put_task_info(MAXFRAME_TASK_GET_DAG_INFO_METHOD, req_data)
         return self._deserial_task_info_result(res, DagInfo)
 
     def cancel_dag(self, dag_id: str) -> DagInfo:
@@ -233,23 +244,33 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             "dag_id": dag_id,
             "output_format": self._output_format,
         }
-        res = self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_CANCEL_DAG_METHOD, json.dumps(req_data)
-        )
+        res = self._put_task_info(MAXFRAME_TASK_CANCEL_DAG_METHOD, req_data)
         return self._deserial_task_info_result(res, DagInfo)
 
     def decref(self, tileable_keys: List[str]) -> None:
         req_data = {
             "tileable_keys": ",".join(tileable_keys),
         }
-        self._instance.put_task_info(
-            self._task_name, MAXFRAME_TASK_DECREF_METHOD, json.dumps(req_data)
-        )
+        self._put_task_info(MAXFRAME_TASK_DECREF_METHOD, req_data)
 
     def get_logview_address(self, dag_id=None, hours=None) -> Optional[str]:
+        """
+        Generate logview address
+
+        Parameters
+        ----------
+            dag_id: id of dag for which dag logview detail page to access
+            hours: hours of the logview address auth limit
+        Returns
+        -------
+            Logview address
+        """
         hours = hours or options.session.logview_hours
-        subquery_suffix = f"&subQuery={dag_id}" if dag_id else ""
-        return self._instance.get_logview_address(hours) + subquery_suffix
+        # notice: maxframe can't reuse subQuery else will conflict with mcqa when fetch resource data,
+        #         added dagId for maxframe so logview backend will return maxframe data format if
+        #         instance and dagId is provided.
+        dag_suffix = f"&dagId={dag_id}" if dag_id else ""
+        return self._instance.get_logview_address(hours) + dag_suffix
 
 
 class MaxFrameTaskSession(MaxFrameSession):

maxframe_client/tests/test_session.py

@@ -23,7 +23,10 @@ from odps import ODPS
 
 import maxframe.dataframe as md
 import maxframe.remote as mr
+from maxframe.config import options
+from maxframe.config.config import option_context
 from maxframe.core import ExecutableTuple, TileableGraph
+from maxframe.errors import NoTaskServerResponseError
 from maxframe.lib.aio import stop_isolation
 from maxframe.protocol import ResultInfo
 from maxframe.serialization import RemoteException
@@ -35,6 +38,7 @@ from maxframe_framedriver.app.tests.test_framedriver_webapp import (  # noqa: F4
 )
 
 from ..clients.framedriver import FrameDriverClient
+from ..session.odps import MaxFrameRestCaller
 
 pytestmark = pytest.mark.maxframe_engine(["MCSQL", "SPE"])
 
@@ -86,11 +90,25 @@ def test_simple_run_dataframe(start_mock_session):
         assert len(dag) == 2
         return await original_submit_dag(self, session_id, dag, managed_input_infos)
 
+    no_task_server_raised = False
+    original_get_dag_info = MaxFrameRestCaller.get_dag_info
+
+    async def patched_get_dag_info(self, dag_id: str):
+        nonlocal no_task_server_raised
+
+        if not no_task_server_raised:
+            no_task_server_raised = True
+            raise NoTaskServerResponseError
+        return await original_get_dag_info(self, dag_id)
+
     df["H"] = "extra_content"
 
     with mock.patch(
         "maxframe_client.clients.framedriver.FrameDriverClient.submit_dag",
         new=patched_submit_dag,
+    ), mock.patch(
+        "maxframe_client.session.odps.MaxFrameRestCaller.get_dag_info",
+        new=patched_get_dag_info,
     ):
         result = df.execute().fetch()
         assert len(result) == 1000
@@ -184,6 +202,24 @@ def test_run_dataframe_from_to_odps_table(start_mock_session):
         odps_entry.delete_table(table_name, if_exists=True)
 
 
+def test_create_session_with_options(framedriver_app):  # noqa: F811
+    odps_entry = ODPS.from_environments()
+    framedriver_addr = f"mf://localhost:{framedriver_app.port}"
+    old_value = options.session.max_alive_seconds
+    session = None
+    try:
+        options.session.max_alive_seconds = 10
+        session = new_session(framedriver_addr, odps_entry=odps_entry)
+        session_id = session.session_id
+        session_conf = framedriver_app.session_manager.get_session_settings(session_id)
+        with option_context(session_conf) as session_options:
+            assert session_options.session.max_alive_seconds == 10
+    finally:
+        options.session.max_alive_seconds = old_value
+        if session is not None:
+            session.destroy()
+
+
 def test_run_and_fetch_series(start_mock_session):
     odps_entry = ODPS.from_environments()
 

maxframe_client/clients/spe.py

@@ -1,104 +0,0 @@
-# Copyright 1999-2024 Alibaba Group Holding Ltd.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Any, Dict, Optional
-
-from tornado import httpclient
-
-from maxframe.core import TileableGraph
-from maxframe.protocol import ExecuteSubDagRequest, ProtocolBody, SubDagInfo
-from maxframe.typing_ import TimeoutType
-from maxframe.utils import (
-    deserialize_serializable,
-    format_timeout_params,
-    serialize_serializable,
-    wait_http_response,
-)
-
-
-class SPEClient:
-    def __init__(
-        self,
-        endpoint: str,
-        session_id: Optional[str] = None,
-        host: str = None,
-    ):
-        self._endpoint = endpoint.rstrip("/")
-        self._session_id = session_id
-        self._headers = {"Host": host}
-
-    @staticmethod
-    def _load_subdag_info(resp: httpclient.HTTPResponse) -> SubDagInfo:
-        res: ProtocolBody[SubDagInfo] = deserialize_serializable(resp.body)
-        return res.body
-
-    async def submit_subdag(
-        self, subdag: TileableGraph, settings: Dict[str, Any] = None
-    ) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags"
-        req_body: ProtocolBody[ExecuteSubDagRequest] = ProtocolBody(
-            body=ExecuteSubDagRequest(dag=subdag, settings=settings),
-        )
-
-        if self._session_id is not None:
-            req_url += f"?session_id={self._session_id}"
-
-        resp = await httpclient.AsyncHTTPClient().fetch(
-            req_url,
-            method="POST",
-            headers=self._headers,
-            body=serialize_serializable(req_body),
-        )
-        return self._load_subdag_info(resp)
-
-    async def get_subdag_info(self, subdag_id: str) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags/{subdag_id}?wait=0"
-        resp = await httpclient.AsyncHTTPClient().fetch(
-            req_url,
-            method="GET",
-            headers=self._headers,
-        )
-        return self._load_subdag_info(resp)
-
-    async def wait_subdag(
-        self, subdag_id: str, wait_timeout: TimeoutType = None
-    ) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags/{subdag_id}"
-        params = format_timeout_params(wait_timeout)
-        try:
-            resp = await wait_http_response(
-                req_url + params,
-                method="GET",
-                headers=self._headers,
-                request_timeout=wait_timeout,
-            )
-            return self._load_subdag_info(resp)
-        except TimeoutError:
-            return await self.get_subdag_info(subdag_id)
-
-    async def cancel_subdag(
-        self, subdag_id: str, wait_timeout: TimeoutType = None
-    ) -> SubDagInfo:
-        req_url = f"{self._endpoint}/api/subdags/{subdag_id}"
-        params = format_timeout_params(wait_timeout)
-        try:
-            resp = await wait_http_response(
-                req_url + params,
-                method="DELETE",
-                headers=self._headers,
-                request_timeout=wait_timeout,
-            )
-            return self._load_subdag_info(resp)
-        except TimeoutError:
-            return await self.get_subdag_info(subdag_id)