maxframe 1.0.0rc4__cp39-cp39-win32.whl → 1.1.1__cp39-cp39-win32.whl

maxframe_client/session/odps.py

@@ -14,6 +14,7 @@
 
 import abc
 import asyncio
+import copy
 import logging
 import time
 import weakref
@@ -25,6 +26,7 @@ import numpy as np
 import pandas as pd
 from odps import ODPS
 from odps import options as odps_options
+from odps.console import in_ipython_frontend
 
 from maxframe.config import options
 from maxframe.core import Entity, TileableGraph, build_fetch, enter_mode
@@ -82,10 +84,21 @@ class MaxFrameServiceCaller(metaclass=abc.ABCMeta):
     def get_settings_to_upload(self) -> Dict[str, Any]:
         sql_settings = (odps_options.sql.settings or {}).copy()
         sql_settings.update(options.sql.settings or {})
-
         quota_name = options.session.quota_name or getattr(
             odps_options, "quota_name", None
         )
+        quota_settings = {
+            sql_settings.get("odps.task.wlm.quota", None),
+            options.spe.task.settings.get("odps.task.wlm.quota", None),
+            options.pythonpack.task.settings.get("odps.task.wlm.quota", None),
+            quota_name,
+        }.difference([None])
+        if len(quota_settings) >= 2:
+            raise ValueError(
+                "Quota settings are conflicting: %s" % ", ".join(sorted(quota_settings))
+            )
+        elif len(quota_settings) == 1:
+            quota_name = quota_settings.pop()
         lifecycle = options.session.table_lifecycle or odps_options.lifecycle
         temp_lifecycle = (
             options.session.temp_table_lifecycle or odps_options.temp_lifecycle
@@ -113,6 +126,8 @@ class MaxFrameServiceCaller(metaclass=abc.ABCMeta):
         mf_settings["session.quota_name"] = quota_name
         if enable_schema is not None:
             mf_settings["session.enable_schema"] = enable_schema
+        if options.session.enable_high_availability is None:
+            mf_settings["session.enable_high_availability"] = not in_ipython_frontend()
         mf_settings["session.default_schema"] = default_schema or "default"
         return mf_settings
 
@@ -184,6 +199,8 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
 
         self._caller = self._create_caller(odps_entry, address, **kwargs)
         self._last_settings = None
+        self._pull_interval = 1 if in_ipython_frontend() else 3
+        self._replace_internal_host = kwargs.get("replace_internal_host", True)
 
     @classmethod
     def _create_caller(
@@ -193,7 +210,7 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
 
     async def _init(self, _address: str):
         session_info = await self.ensure_async_call(self._caller.create_session)
-        self._last_settings = self._caller.get_settings_to_upload()
+        self._last_settings = copy.deepcopy(self._caller.get_settings_to_upload())
         self._session_id = session_info.session_id
         await self._show_logview_address()
 
@@ -250,7 +267,12 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
         self, t: TileableType
     ) -> Optional[TileableType]:
         vol_name = build_session_volume_name(self.session_id)
-        writer = ODPSVolumeWriter(self._odps_entry, vol_name, t.key)
+        writer = ODPSVolumeWriter(
+            self._odps_entry,
+            vol_name,
+            t.key,
+            replace_internal_host=self._replace_internal_host,
+        )
         io_handler = get_object_io_handler(t)
         io_handler().write_object(writer, t, t.op.data)
         return build_fetch(t).data
@@ -318,9 +340,14 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
     def _get_diff_settings(self) -> Dict[str, Any]:
         new_settings = self._caller.get_settings_to_upload()
         if not self._last_settings:  # pragma: no cover
-            self._last_settings = new_settings
+            self._last_settings = copy.deepcopy(new_settings)
             return new_settings
 
+        if self._last_settings.get("session.quota_name", None) != new_settings.get(
+            "session.quota_name", None
+        ):
+            raise ValueError("Quota name cannot be changed after sessions are created")
+
         update = dict()
         for k in new_settings.keys():
             old_item = self._last_settings.get(k)
@@ -330,7 +357,7 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
                     update[k] = new_item
             except:  # noqa: E722  # nosec  # pylint: disable=bare-except
                 update[k] = new_item
-        self._last_settings = new_settings
+        self._last_settings = copy.deepcopy(new_settings)
         return update
 
     async def execute(self, *tileables, **kwargs) -> ExecutionInfo:
@@ -379,18 +406,18 @@ class MaxFrameSession(ToThreadMixin, IsolatedAsyncSession):
         start_time = time.time()
         session_id = dag_info.session_id
         dag_id = dag_info.dag_id
-        wait_timeout = 10
         server_no_response_time = None
         with enter_mode(build=True, kernel=True):
             key_to_tileables = {t.key: t for t in tileables}
-
+            timeout_val = 0.1
             try:
                 while True:
                     elapsed_time = time.time() - start_time
+                    next_timeout_val = min(timeout_val * 2, self._pull_interval)
                     timeout_val = (
-                        min(self.timeout - elapsed_time, wait_timeout)
+                        min(self.timeout - elapsed_time, next_timeout_val)
                         if self.timeout
-                        else wait_timeout
+                        else next_timeout_val
                     )
                     if timeout_val <= 0:
                         raise TimeoutError("Running DAG timed out")
@@ -584,7 +611,9 @@ class MaxFrameRestCaller(MaxFrameServiceCaller):
         managed_input_infos: Dict[str, ResultInfo] = None,
         new_settings: Dict[str, Any] = None,
     ) -> DagInfo:
-        return await self._client.submit_dag(self._session_id, dag, managed_input_infos)
+        return await self._client.submit_dag(
+            self._session_id, dag, managed_input_infos, new_settings=new_settings
+        )
 
     async def get_dag_info(self, dag_id: str) -> DagInfo:
         return await self._client.get_dag_info(self._session_id, dag_id)
@@ -617,7 +646,7 @@ class MaxFrameRestSession(MaxFrameSession):
         real_endpoint = address.replace(f"{parsed_endpoint.scheme}://", f"{scheme}://")
 
         super().__init__(
-            real_endpoint, session_id, odps_entry=odps_entry, timeout=timeout
+            real_endpoint, session_id, odps_entry=odps_entry, timeout=timeout, **kwargs
         )
 
     @classmethod

maxframe_client/session/task.py

@@ -21,15 +21,9 @@ from typing import Any, Dict, List, Optional, Type, Union
 import msgpack
 from odps import ODPS
 from odps import options as odps_options
-from odps.errors import parse_instance_error
+from odps.errors import EmptyTaskInfoError, parse_instance_error
 from odps.models import Instance, MaxFrameTask
 
-try:
-    from odps.errors import EmptyTaskInfoError
-except ImportError:  # pragma: no cover
-    # todo remove when pyodps>=0.12.0 is enforced
-    EmptyTaskInfoError = type("EmptyTaskInfoError", (Exception,), {})
-
 from maxframe.config import options
 from maxframe.core import TileableGraph
 from maxframe.errors import NoTaskServerResponseError, SessionAlreadyClosedError
@@ -131,15 +125,14 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
             )
 
     def _create_maxframe_task(self) -> MaxFrameTask:
-        task = MaxFrameTask(
-            name=self._task_name,
-            major_version=self._major_version,
-            service_endpoint=self._odps_entry.endpoint,
-        )
+        task = MaxFrameTask(name=self._task_name, major_version=self._major_version)
+        mf_settings = self.get_settings_to_upload()
         mf_opts = {
-            "odps.maxframe.settings": json.dumps(self.get_settings_to_upload()),
+            "odps.maxframe.settings": json.dumps(mf_settings),
             "odps.maxframe.output_format": self._output_format,
         }
+        if mf_settings.get("session.quota_name", None):
+            mf_opts["odps.task.wlm.quota"] = mf_settings["session.quota_name"]
         if mf_version:
             mf_opts["odps.maxframe.client_version"] = mf_version
         task.update_settings(mf_opts)
@@ -200,18 +193,6 @@ class MaxFrameInstanceCaller(MaxFrameServiceCaller):
                     json.dumps(json_data),
                     raise_empty=True,
                 )
-            except TypeError:  # pragma: no cover
-                # todo remove when pyodps>=0.12.0 is enforced
-                resp_data = self._instance.put_task_info(
-                    self._task_name, method_name, json.dumps(json_data)
-                )
-                if resp_data:
-                    return resp_data
-                else:
-                    raise NoTaskServerResponseError(
-                        f"No response for request {method_name}. "
-                        f"Instance ID: {self._instance.id}"
-                    )
             except EmptyTaskInfoError as ex:
                 # retry when server returns HTTP 204, which is designed for retry
                 if ex.code != 204 or trial >= EMPTY_RESPONSE_RETRY_COUNT - 1:

maxframe_client/session/tests/test_task.py

@@ -11,17 +11,20 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import json
 import os
 
 import mock
+import pytest
 from defusedxml import ElementTree
 from odps import ODPS
 from odps import options as odps_options
 
+from maxframe import options
+from maxframe.config import option_context
+
 from ...session.consts import MAXFRAME_OUTPUT_JSON_FORMAT
-from ...session.task import MaxFrameInstanceCaller, MaxFrameTask
+from ...session.task import MaxFrameInstanceCaller, MaxFrameTask, MaxFrameTaskSession
 
 expected_file_dir = os.path.join(os.path.dirname(__file__), "expected-data")
 
@@ -53,10 +56,6 @@ def test_maxframe_instance_caller_creating_session():
         assert property_node.find("Name").text == "settings"
         setting_dict = json.loads(property_node.find("Value").text)
         assert setting_dict["odps.task.major.version"] == "test_version"
-        assert (
-            setting_dict["odps.service.endpoint"]
-            == "http://100.69.248.78:8002/odps_dailyrunnew"
-        )
 
         assert setting_dict["odps.maxframe.output_format"] == "json"
         maxframe_setting_dict = json.loads(setting_dict["odps.maxframe.settings"])
@@ -83,3 +82,33 @@ def test_maxframe_instance_caller_creating_session():
         finally:
             odps_options.priority = old_priority
             odps_options.get_priority = old_get_priority
+
+
+@pytest.mark.asyncio
+async def test_session_quota_flag_valid():
+    def mock_create(self, task: MaxFrameTask, **kwargs):
+        assert task.properties["settings"]
+        task_settings = json.loads(task.properties["settings"])
+        assert task_settings["odps.task.wlm.quota"] == "session_quota"
+
+    with mock.patch.multiple(
+        target="maxframe_client.session.task.MaxFrameInstanceCaller",
+        _wait_instance_task_ready=mock.DEFAULT,
+        get_session=mock.DEFAULT,
+        get_logview_address=mock.DEFAULT,
+    ), mock.patch("odps.models.instances.BaseInstances.create", mock_create):
+        with option_context({"session.quota_name": "session_quota"}):
+            with pytest.raises(ValueError):
+                options.sql.settings["odps.task.wlm.quota"] = "session_quota2"
+                await MaxFrameTaskSession.init(
+                    address="test", odps_entry=ODPS.from_environments()
+                )
+            options.sql.settings["odps.task.wlm.quota"] = "session_quota"
+            mf_task_session = await MaxFrameTaskSession.init(
+                address="test", odps_entry=ODPS.from_environments()
+            )
+            with pytest.raises(ValueError):
+                options.sql.settings["odps.task.wlm.quota"] = "session_quota2"
+                mf_task_session._get_diff_settings()
+            options.sql.settings["odps.task.wlm.quota"] = "session_quota"
+            mf_task_session._get_diff_settings()

maxframe_client/tests/test_session.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 import time
-from typing import Dict
+from typing import Any, Dict
 
 import mock
 import numpy as np
@@ -31,7 +31,7 @@ from maxframe.lib.aio import stop_isolation
 from maxframe.protocol import ResultInfo
 from maxframe.serialization import RemoteException
 from maxframe.session import new_session
-from maxframe.tests.utils import tn
+from maxframe.tests.utils import ensure_table_deleted, tn
 from maxframe.utils import build_temp_table_name
 from maxframe_framedriver.app.tests.test_framedriver_webapp import (  # noqa: F401
     framedriver_app,
@@ -86,9 +86,12 @@ def test_simple_run_dataframe(start_mock_session):
         session_id: str,
         dag: TileableGraph,
         managed_input_infos: Dict[str, ResultInfo] = None,
+        new_settings: Dict[str, Any] = None,
     ):
         assert len(dag) == 2
-        return await original_submit_dag(self, session_id, dag, managed_input_infos)
+        return await original_submit_dag(
+            self, session_id, dag, managed_input_infos, new_settings
+        )
 
     no_task_server_raised = False
     original_get_dag_info = MaxFrameRestCaller.get_dag_info
@@ -130,11 +133,10 @@ def test_simple_run_dataframe(start_mock_session):
     )
     assert odps_entry.exist_table(build_temp_table_name(start_mock_session, key))
     del df
-    time.sleep(5)
-    assert not odps_entry.exist_table(
-        build_temp_table_name(start_mock_session, intermediate_key)
+    ensure_table_deleted(
+        odps_entry, build_temp_table_name(start_mock_session, intermediate_key)
     )
-    assert not odps_entry.exist_table(build_temp_table_name(start_mock_session, key))
+    ensure_table_deleted(odps_entry, build_temp_table_name(start_mock_session, key))
 
 
 def test_run_and_fetch_slice(start_mock_session):
@@ -166,6 +168,25 @@ def test_run_empty_table(start_mock_session):
     empty_table.drop()
 
 
+def test_run_odps_query_without_schema(start_mock_session):
+    odps_entry = ODPS.from_environments()
+
+    table_name = tn("test_query_without_schema")
+    odps_entry.delete_table(table_name, if_exists=True)
+    test_table = odps_entry.create_table(table_name, "a double, b double", lifecycle=1)
+
+    with test_table.open_writer() as writer:
+        writer.write([123, 456])
+
+    df = md.read_odps_query(
+        f"select a, b, a + b as `special: name` from {table_name}", skip_schema=True
+    )
+    executed = df.execute().fetch()
+    assert len(executed.dtypes) == 3
+
+    test_table.drop()
+
+
 def test_run_dataframe_with_pd_source(start_mock_session):
     odps_entry = ODPS.from_environments()
 
@@ -246,9 +267,8 @@ def test_run_and_fetch_series(start_mock_session):
         pd.testing.assert_series_equal(pd_result, result)
 
         del s1
-        time.sleep(5)
-        assert not odps_entry.exist_table(
-            build_temp_table_name(start_mock_session, src_key)
+        ensure_table_deleted(
+            odps_entry, build_temp_table_name(start_mock_session, src_key)
         )
     finally:
         odps_entry.delete_table(