deepfos 1.1.31__tar.gz → 1.1.32__tar.gz

{deepfos-1.1.31 → deepfos-1.1.32}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.31
+Version: 1.1.32
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.31 → deepfos-1.1.32}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-03-26T03:15:02+0000",
+ "date": "2024-04-08T07:12:04+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "52195bbba97196b1fd930df4bc7c5def16ab0a90",
- "version": "1.1.31"
+ "full-revisionid": "2eb652409a767c757a4324b2ceb60a88d649f355",
+ "version": "1.1.32"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.31 → deepfos-1.1.32}/deepfos/db/edb.py

@@ -1,7 +1,14 @@
-from edgedb import AsyncIOClient, errors, enums, describe
-from edgedb.abstract import DescribeContext
-from edgedb.asyncio_client import AsyncIOConnection # noqa
-from edgedb.protocol import protocol
+import asyncio
+
+from edgedb import AsyncIOClient, errors, enums, describe, abstract
+from edgedb.abstract import (
+    DescribeContext, QueryContext, QueryWithArgs,
+    _query_opts    # noqa
+)
+from edgedb.asyncio_client import (
+    AsyncIOConnection, AsyncIOIteration,    # noqa
+    AsyncIORetry
+)
 
 from deepfos import OPTION
 
@@ -28,27 +35,11 @@ def collect_output_frame(ele_desc: describe.AnyType):
 
 
 class _AsyncEdgeDBConnection(AsyncIOConnection):
-    async def _execute(self, execute_context) -> None:
-        await self._protocol.execute(
-            query=execute_context.query.query,
-            args=execute_context.query.args,
-            kwargs=execute_context.query.kwargs,
-            reg=execute_context.cache.codecs_registry,
-            qc=execute_context.cache.query_cache,
-            output_format=protocol.OutputFormat.NONE,
-            allow_capabilities=enums.Capability.MODIFICATIONS,
-            state=(
-                execute_context.state.as_dict()
-                if execute_context.state else None
-            ),
-        )
-
-    async def raw_query(self, query_context):
+    async def raw_query(self, query_context, capabilities=enums.Capability.NONE):
         if self.is_closed():
             await self.connect()
 
         reconnect = False
-        capabilities = None
         i = 0
         args = dict(
             query=query_context.query.query,
@@ -59,7 +50,7 @@ class _AsyncEdgeDBConnection(AsyncIOConnection):
             output_format=query_context.query_options.output_format,
             expect_one=query_context.query_options.expect_one,
             required_one=query_context.query_options.required_one,
-            allow_capabilities=enums.Capability.NONE,
+            allow_capabilities=capabilities,
         )
         if query_context.state is not None:
             args["state"] = query_context.state.as_dict()
@@ -123,6 +114,43 @@ class _AsyncEdgeDBConnection(AsyncIOConnection):
                 reconnect = self.is_closed()
 
 
+class _AsyncIOIteration(AsyncIOIteration):
+    async def _execute(self, query_context: abstract.QueryContext):
+        with self._exclusive():
+            await self._ensure_transaction()
+            return await self._connection.raw_query(
+                query_context, enums.Capability.MODIFICATIONS
+            )
+
+    async def execute(self, commands: str, *args, **kwargs):
+        return await self._execute(QueryContext(
+            query=QueryWithArgs(commands, args, kwargs),
+            cache=self._get_query_cache(),
+            query_options=_query_opts,
+            retry_options=self._get_retry_options(),
+            state=self._get_state(),
+        ))
+
+
+class _AsyncIORetry(AsyncIORetry):
+    async def __anext__(self):
+        # Note: when changing this code consider also
+        # updating Retry.__next__.
+        if self._done:
+            raise StopAsyncIteration
+        if self._next_backoff:
+            await asyncio.sleep(self._next_backoff)
+        self._done = True
+        iteration = _AsyncIOIteration(self, self._owner, self._iteration)
+        self._iteration += 1
+        return iteration
+
+
+class _AsyncIOClient(AsyncIOClient):
+    def transaction(self) -> _AsyncIORetry:
+        return _AsyncIORetry(self)
+
+
 # All deprecated space in v3dev & v3test & alpha
 deprecated_space = [
     'ulqtqb',
@@ -145,7 +173,7 @@ deprecated_space = [
 def create_async_client(default_module=None):
     space = OPTION.api.header['space']
     dbname = None if space in deprecated_space else f"deepmodel_space{space}"
-    cli = AsyncIOClient(
+    cli = _AsyncIOClient(
         connection_class=_AsyncEdgeDBConnection,
         max_concurrency=None,
         dsn=OPTION.edgedb.dsn,

{deepfos-1.1.31 → deepfos-1.1.32}/deepfos/element/deepmodel.py

@@ -243,7 +243,7 @@ class BaseField(FieldInfo):
         # df[self.col_name] = df[self.col_name].astype(self.dtype, errors='ignore')
         pass
 
-    def cast(self, df: pd.DataFrame, field_name: str):
+    def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
         """
         对 :class:`Dataframe` 对应的列作类型转换。
         一般在获取 :class:`Dataframe` 时使用。
@@ -261,7 +261,7 @@ class FieldDateTime(BaseField):
     def extra_fit(self, df: pd.DataFrame, field_name: str):
         df[field_name] = df[field_name].apply(self.format_datetime)
 
-    def cast(self, df: pd.DataFrame, field_name: str):
+    def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
         df[field_name] = pd.to_datetime(df[field_name], errors='ignore')
 
 
@@ -281,11 +281,11 @@ class FieldJson(BaseField):
             return data
         return json.dumps(data)
 
-    def cast(self, df: pd.DataFrame, field_name: str):
+    def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
         # std::json needed to be cast only when data is from http
         # since json value will be converted to json string(type: str)
         # in edgedb python protocol
-        if not ONLINE_MODE:
+        if not direct_access:
             df[field_name] = df[field_name].apply(self.format_json)
 
 
@@ -295,17 +295,17 @@ class FieldInt(FieldString):
             return pd.NA
         return str(int(data))
 
-    def cast(self, df: pd.DataFrame, field_name: str):
+    def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
         df[field_name] = df[field_name].astype(pd.Int64Dtype(), errors='ignore')
 
 
 class FieldDecimal(FieldString):
-    def cast(self, df: pd.DataFrame, field_name: str):
+    def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
         df[field_name] = df[field_name].astype(pd.Float64Dtype(), errors='ignore')
 
 
 class FieldBool(BaseField):
-    def cast(self, df: pd.DataFrame, field_name: str):
+    def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
         df[field_name] = df[field_name].astype(pd.BooleanDtype(), errors='ignore')
 
 
@@ -362,10 +362,10 @@ class ObjectStructure:
 
         return df[valid_fields]
 
-    def cast(self, df: pd.DataFrame):
+    def cast(self, df: pd.DataFrame, direct_access: bool = True):
         for field in df.columns:
             if field in self.fields:
-                self.fields[field].cast(df, field)
+                self.fields[field].cast(df, field, direct_access)
 
 
 def _iter_link_prop_assign(link, business_key, prop_name, prop_type, is_multi):
@@ -567,23 +567,25 @@ class _TxnConfig:
         self.txn_support = False
 
 
-if OPTION.edgedb.dsn:
-    ONLINE_MODE = True
-else:
-    ONLINE_MODE = False
-
-
 # -----------------------------------------------------------------------------
 # core
 class AsyncDeepModel(ElementBase[DeepModelAPI]):
     """DeepModel"""
 
-    def __init__(self):  # noqa
+    def __init__(self, direct_access: bool=True):  # noqa
         self._txn_ = ContextVar('QLTXN')
         self.appmodule = f"app{OPTION.api.header['app']}"
         self.spacemodule = f"space{OPTION.api.header['space']}"
-        if ONLINE_MODE:
+        self.direct_access = direct_access
+        if direct_access:
             self.client = create_async_client(default_module=self.appmodule)
+            if user_id := OPTION.api.header.get('user'):
+                self.client = self.client.with_globals(
+                    **{
+                        f'{self.spacemodule}::current_user_id':
+                        user_id
+                    }
+                )
         else:
             self.client = None
         self.alias = AliasGenerator()
@@ -690,7 +692,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             
         """
 
-        if ONLINE_MODE:
+        if self.direct_access:
             logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
             async with self.client as cli:
                 _, result = await cli.query(ql, **kwargs)
@@ -726,7 +728,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             :func:`query_object`, 执行ql查询语句，得到原始结果返回
 
         """
-        if ONLINE_MODE:
+        if self.direct_access:
             logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
             async with self.client as cli:
                 frame_desc, result = await cli.query(ql, **kwargs)
@@ -775,7 +777,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             :func:`query_object`, 执行ql查询语句，得到原始结果返回
         
         """
-        if ONLINE_MODE:
+        if self.direct_access:
             async with self.client as cli:
                 frame_desc, data = await cli.query(ql, **kwargs)
 
@@ -803,7 +805,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         if data.empty:
             return pd.DataFrame(columns=structure.fields.keys())
 
-        structure.cast(data)
+        structure.cast(data, self.direct_access)
         return data
 
     query.__doc__ = query.__doc__ + DOC_ARGS_KWARGS
@@ -811,7 +813,11 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
     query_df.__doc__ = query_df.__doc__ + DOC_ARGS_KWARGS
 
     @txn_support
-    async def execute(self, qls: Union[str, List[str], List[QueryWithArgs]], **kwargs):
+    async def execute(
+        self,
+        qls: Union[str, List[str], List[QueryWithArgs]],
+        **kwargs
+    ) -> Optional[List]:
         """以事务执行多句ql
 
         Args:
@@ -828,10 +834,12 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             seen_kwargs_key = set()
             for ql in qls:
                 if isinstance(ql, QueryWithArgs):
-                    if not ONLINE_MODE and ql.kwargs and seen_kwargs_key.intersection(
-                        ql.kwargs.keys()
+                    if (
+                        not self.direct_access
+                        and ql.kwargs
+                        and seen_kwargs_key.intersection(ql.kwargs.keys())
                     ):
-                        raise NotImplementedError('线下模式不支持重名variables')
+                        raise NotImplementedError('非直连模式不支持重名variables')
 
                     qls_with_args.append(ql)
                     if ql.kwargs:
@@ -842,19 +850,19 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                 else:
                     raise TypeError(f'qls参数中出现类型非法成员：{type(ql)}')
 
-        await self._maybe_exec_qls(qls_with_args)
+        return await self._maybe_exec_qls(qls_with_args)
 
     execute.__doc__ = execute.__doc__ + DOC_ARGS_KWARGS
 
-    async def _execute(self, qls_with_args: List[QueryWithArgs]):
+    async def _execute(self, qls_with_args: List[QueryWithArgs]) -> List:
         self.alias.reset(BATCH_INSERT_KW)
-        if not ONLINE_MODE:
+        if not self.direct_access:
             kwargs = {}
             seen_kwargs_key = set()
 
             for ql in qls_with_args:
                 if ql.kwargs and seen_kwargs_key.intersection(ql.kwargs.keys()):
-                    raise NotImplementedError('线下模式不支持重名variables')
+                    raise NotImplementedError('非直连模式不支持重名variables')
                 if ql.kwargs:
                     kwargs.update(ql.kwargs)
                     seen_kwargs_key = seen_kwargs_key.union(ql.kwargs.keys())
@@ -869,9 +877,11 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                 query=commands,
                 variables=self._prepare_variables(kwargs)
             )
-            self._maybe_handle_error(res.get('json'))
-            return
+            affected = res.get('json')
+            self._maybe_handle_error(affected)
+            return affected
 
+        result = []
         async with self.client as cli:
             async for tx in cli.transaction():
                 async with tx:
@@ -879,7 +889,13 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                         logger.opt(lazy=True).debug(
                             f"Execute QL: [{ql.commands}], \nkwargs: [{ql.kwargs}]."
                         )
-                        await tx.execute(ql.commands, **ql.kwargs)
+                        desc, affected = await tx.execute(ql.commands, **ql.kwargs)
+                        result.append(serutils.serialize(
+                            affected, ctx=serutils.Context(frame_desc=desc)
+                        ))
+        if len(result) == 1:
+            return result[0]
+        return result
 
     @staticmethod
     def _maybe_handle_error(res):
@@ -891,14 +907,17 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             ex_code = int(error['code'])
             raise edgedb.EdgeDBError._from_code(ex_code, ex_msg)  # noqa
 
-    async def _maybe_exec_qls(self, qls_with_args: List[QueryWithArgs]):
+    async def _maybe_exec_qls(
+        self,
+        qls_with_args: List[QueryWithArgs]
+    ) -> Optional[List]:
         txn_conf = self._safe_get_txn_conf()
 
         if txn_conf.in_txn and self._txn_support_:
             txn_conf.qls.extend(qls_with_args)
             return
 
-        await self._execute(qls_with_args)
+        return await self._execute(qls_with_args)
 
     @txn_support
     async def insert_df(
@@ -909,7 +928,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         chunksize: int = 500,
         enable_upsert: bool = False,
         update_fields: List[str] = None,
-    ):
+    ) -> None:
         """以事务执行基于DataFrame字段信息的批量插入数据
 
         Args:
@@ -1119,7 +1138,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                 kwargs={kw_name: part.to_json(orient='records')}
             ))
 
-    def get_object(self, object_name):
+    def get_object(self, object_name) -> edgedb.Object:
         if object_name in self.user_objects:
             obj = self.user_objects[object_name]
         elif object_name in self.system_objects:
@@ -1137,7 +1156,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         obj: Union[ObjectTypeFrame, TargetField],
         source_name: str = None,
         field_name: str = None
-    ):
+    ) -> str:
         # 如可在object结构的annotations中取业务主键，则优先取，否则走接口
         if obj.info and BUSINESS_KEY in obj.info:
             return obj.info[BUSINESS_KEY]
@@ -1262,6 +1281,36 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         finally:
             self._txn_.set(_TxnConfig())
 
+    @contextmanager
+    def with_globals(self, globals_):
+        if not self.direct_access:
+            try:
+                yield
+            finally:
+                raise NotImplemented('非直连模式不支持设置state信息')
+        else:
+            bak_cli = self.client
+            try:
+                self.client = self.client.with_globals(**globals_)
+                yield
+            finally:
+                self.client = bak_cli
+
+    @contextmanager
+    def without_globals(self, *global_names):
+        if not self.direct_access:
+            try:
+                yield
+            finally:
+                raise NotImplemented('非直连模式不支持设置state信息')
+        else:
+            bak_cli = self.client
+            try:
+                self.client = self.client.without_globals(*global_names)
+                yield
+            finally:
+                self.client = bak_cli
+
 
 class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
     synchronize = ('query_object', 'query', 'query_df', 'execute', 'insert_df')
@@ -1276,7 +1325,11 @@ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
         def query_df(self, ql: str, **kwargs) -> pd.DataFrame:
             ...
 
-        def execute(self, qls: Union[str, List[str], List[QueryWithArgs]], **kwargs):
+        def execute(
+            self,
+            qls: Union[str, List[str], List[QueryWithArgs]],
+            **kwargs
+        ) -> Optional[List]:
             ...
 
         def insert_df(
@@ -1287,7 +1340,7 @@ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
             chunksize: int = 500,
             enable_upsert: bool = False,
             update_fields: List[str] = None,
-        ):
+        ) -> None:
             ...
 
     @contextmanager

{deepfos-1.1.31 → deepfos-1.1.32}/deepfos/lib/eureka.py

@@ -35,8 +35,13 @@ class EurekaCli(ServiceDiscovery):
 
     # noinspection PyAttributeOutsideInit
     def setup(self):
-        self._app_url = concat_url(OPTION.server.eureka, 'apps')
-        self._delta_url = concat_url(OPTION.server.eureka, 'apps/delta')
+        from deepfos.lib.discovery import CACHE_STRATEGY, RankedCache
+        base_url = CACHE_STRATEGY.get(
+            OPTION.discovery.cache_strategy, RankedCache
+        )()
+        for url in OPTION.server.eureka.split(','):
+            base_url.add(url)
+        self.base_url = base_url
 
     async def on_interval(self):
         if self.server_cache:  # 有全量数据时-保持增量更新，否则-全量更新
@@ -63,11 +68,26 @@ class EurekaCli(ServiceDiscovery):
     async def update_instance_cache(self, server_name: str):
         pass
 
+    async def _get_xml(self, end_point: str):
+        if not self.base_url:
+            # Maybe all deleted by punish
+            self.setup()
+
+        url = self.base_url.pick_best()
+        try:
+            resp = await AioHttpCli.get(concat_url(url, end_point))
+            raw_xml = await resp.text()
+            xml = ElementTree.fromstring(raw_xml)
+        except Exception:
+            self.base_url.punish(url)
+            raise
+        else:
+            self.base_url.reward(url)
+        return xml
+
     async def _get_full_apps(self):
         """全量获取eureka服务列表，并更新到缓存中"""
-        resp = await AioHttpCli.get(self._app_url)
-        raw_xml = await resp.text()
-        xml = ElementTree.fromstring(raw_xml)
+        xml = await self._get_xml('apps')
 
         # 全量更新：确保拿到数据后，先清缓存
         if xml.find('apps__hashcode').text:
@@ -91,9 +111,7 @@ class EurekaCli(ServiceDiscovery):
 
     async def get_delta_apps(self):
         """增量获取eureka服务列表，并更新到缓存中"""
-        resp = await AioHttpCli.get(self._delta_url)
-        raw_xml = await resp.text()
-        xml = ElementTree.fromstring(raw_xml)
+        xml = await self._get_xml('apps/delta')
 
         server_memo = self.server_cache
         if (

{deepfos-1.1.31 → deepfos-1.1.32}/deepfos/lib/sysutils.py

@@ -225,10 +225,17 @@ def export_file_for_download(file_name: str, file: Union[str, bytes, TextIOWrapp
 
             # . 提供buffer
             import io
+            import pandas as pd
 
             buffer = io.BytesIO()
-            buffer.write(b'8888')
-            export_file_for_download('t4.txt', buffer.getbuffer())
+            # 将dataframe内容写入buffer
+            with pd.ExcelWriter(buffer, engine="openpyxl") as writer:
+                  pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]}).to_excel(
+                    writer, index=False, encoding='utf-8'
+                  )
+
+            # 上传至下载中心
+            export_file_for_download('out.xlsx', buffer.getbuffer())
 
 
     """

{deepfos-1.1.31 → deepfos-1.1.32}/deepfos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.31
+Version: 1.1.32
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team