deepfos 1.1.52__tar.gz → 1.1.53__tar.gz

{deepfos-1.1.52 → deepfos-1.1.53}/CHANGELOG.md

@@ -1,3 +1,13 @@
+## [1.1.53] - 2025-02-25
+
+### 新增
+
+* 封装数据流3.0组件元素
+
+### 变更
+
+* 依赖的三方库版本升级
+
 ## [1.1.52] - 2025-01-07
 
 ### 变更

{deepfos-1.1.52 → deepfos-1.1.53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.52
+Version: 1.1.53
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team
@@ -12,19 +12,21 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.8.3
-Requires-Dist: pandas==1.2.0
-Requires-Dist: requests==2.25.1
+Requires-Dist: pandas
+Requires-Dist: requests~=2.26
 Requires-Dist: pymysql==1.1.1
-Requires-Dist: aiohttp==3.9.0
-Requires-Dist: pydantic==1.8.2
-Requires-Dist: cachetools==4.2.2
+Requires-Dist: aiohttp~=3.9.5
+Requires-Dist: pydantic<2.0.0,>=1.10.0
+Requires-Dist: cachetools<6,>=5.0.0
 Requires-Dist: loguru==0.5.3
 Requires-Dist: PyPika==0.48.8
-Requires-Dist: aiomysql==0.2.0
-Requires-Dist: clickhouse-driver==0.2.2
+Requires-Dist: aiomysql~=0.2.0
+Requires-Dist: clickhouse-driver~=0.2.9
 Requires-Dist: aioredis==2.0.1
 Requires-Dist: redis==4.5.3
-Requires-Dist: deepfos-ipc~=1.2.3
-Requires-Dist: typing-extensions==4.5.0
+Requires-Dist: nest-asyncio==1.5.4
+Requires-Dist: deepfos-ipc~=1.3.1
+Requires-Dist: typing-extensions>=4.8.0
 Requires-Dist: edgedb~=1.4.0
 Requires-Dist: asyncpg~=0.29.0
+Requires-Dist: numpy<2.0.0

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-01-08T03:11:42+0000",
+ "date": "2025-02-27T06:36:31+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "da07717ad109df0e4a31cc93ce68ad1a37470045",
- "version": "1.1.52"
+ "full-revisionid": "8a59c898c75628d287c6ef2d8889139be57b7cfe",
+ "version": "1.1.53"
 }
 '''  # END VERSION_JSON
 

deepfos-1.1.53/deepfos/api/deep_pipeline.py

@@ -0,0 +1,61 @@
+from typing import Union, Any, Awaitable
+
+from deepfos.lib.decorator import cached_property
+from .base import DynamicRootAPI, ChildAPI, get, post
+from .models.deep_pipeline import *
+
+__all__ = ['DeepPipelineAPI']
+
+
+class RunAPI(ChildAPI):
+    """数据流的执行、获取执行信息等逻辑"""
+    endpoint = '/'
+
+    @post('run')
+    def run_async(self, run_info: RunInfo) -> Union[str, Awaitable[str]]:
+        """
+        异步执行数据流
+        异步执行数据流，返回任务id
+        """
+        return {'body': run_info}
+
+    @get('run')
+    def result(self, task_id: str, timeout: int = None) -> Union[Any, Awaitable[Any]]:
+        """获取执行结果"""
+        return {'param': {'timeout': timeout}, 'path': f'{task_id}/result'}
+
+
+class FlowAPI(ChildAPI):
+    """数据流定义相关接口"""
+    endpoint = '/flow'
+
+    @get('')
+    def open(self, element_name: str, folder_id: str, version: str) -> Union[FlowInfo, Awaitable[FlowInfo]]:
+        """查看数据流"""
+        return {
+            'param': {
+                'elementName': element_name,
+                'folderId': folder_id,
+                'version': version
+            }
+        }
+
+
+class DeepPipelineAPI(DynamicRootAPI, builtin=True):
+    """DeepPipeline组件接口"""
+    module_type = 'DPL'
+    default_version = (3, 0)
+    multi_version = False
+    cls_name = 'DeepPipelineAPI'
+    module_name = 'deepfos.api.deep_pipeline'
+    api_version = (3, 0)
+
+    @cached_property
+    def run(self) -> RunAPI:
+        """数据流的执行、获取执行信息等逻辑"""
+        return RunAPI(self)
+
+    @cached_property
+    def flow(self) -> FlowAPI:
+        """数据流定义相关接口"""
+        return FlowAPI(self)

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/api/deepconnector.py

@@ -7,7 +7,11 @@ from .base import ChildAPI, get, RootAPI
 from .models.deepconnector import *
 
 
+__all__ = ['DeepConnectorAPI']
+
+
 class DataSourceAPI(ChildAPI):
+    """连接信息相关接口"""
     endpoint = '/apis/v3/ds/spaces/{space}/apps/{app}'
 
     @get('connection-info', data_wrapped=False)
@@ -21,10 +25,12 @@ class DataSourceAPI(ChildAPI):
 
 
 class DeepConnectorAPI(RootAPI):
+    """连接器组件接口"""
     prefix = lambda: 'http://deep-connector-server'
     url_need_format = True
     module_type = 'CONN'
 
     @cached_property
     def datasource(self) -> DataSourceAPI:
+        """连接信息相关接口"""
         return DataSourceAPI(self)

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/api/deepmodel.py

@@ -4,8 +4,11 @@ from deepfos.lib.decorator import cached_property
 from .base import DynamicRootAPI, ChildAPI, get, post
 from .models.deepmodel import *
 
+__all__ = ['DeepModelAPI']
+
 
 class ObjectAPI(ChildAPI):
+    """模型对象相关"""
     endpoint = '/object'
 
     @get('all/get')
@@ -22,6 +25,7 @@ class ObjectAPI(ChildAPI):
 
 
 class RuleAPI(ChildAPI):
+    """规则清单相关"""
     endpoint = '/rules'
 
     @get('/')
@@ -38,6 +42,7 @@ class RuleAPI(ChildAPI):
 
 
 class SeqAPI(ChildAPI):
+    """序列生成相关"""
     endpoint = '/sequence'
 
     @post('/instance/edit')
@@ -79,6 +84,7 @@ class SeqAPI(ChildAPI):
 
 
 class ShardingAPI(ChildAPI):
+    """分库信息"""
     endpoint = '/sharding'
 
     @get('database')
@@ -87,6 +93,7 @@ class ShardingAPI(ChildAPI):
 
 
 class DeepQLAPI(ChildAPI):
+    """查询器"""
     endpoint = '/public/deepql/actions'
 
     @post('query')
@@ -99,6 +106,7 @@ class DeepQLAPI(ChildAPI):
 
 
 class PresentationAPI(ChildAPI):
+    """展示层"""
     endpoint = '/presentation-layer'
 
     @get('ql-selector/ql-record-info')
@@ -127,6 +135,7 @@ class PresentationAPI(ChildAPI):
 
 
 class ExtraAPI(ChildAPI):
+    """其他接口"""
     endpoint = '/'
 
     @get('git-version')
@@ -135,6 +144,7 @@ class ExtraAPI(ChildAPI):
 
 
 class DeepModelAPI(DynamicRootAPI, builtin=True):
+    """DeepModel组件接口"""
     module_type = 'DM'
     default_version = (1, 0)
     multi_version = False
@@ -144,28 +154,35 @@ class DeepModelAPI(DynamicRootAPI, builtin=True):
 
     @cached_property
     def object(self) -> ObjectAPI:
+        """模型对象相关"""
         return ObjectAPI(self)
 
     @cached_property
     def deepql(self) -> DeepQLAPI:
+        """查询器"""
         return DeepQLAPI(self)
 
     @cached_property
     def rule(self) -> RuleAPI:
+        """规则清单相关"""
         return RuleAPI(self)
 
     @cached_property
     def seq(self) -> SeqAPI:
+        """序列生成相关"""
         return SeqAPI(self)
 
     @cached_property
     def sharding(self) -> ShardingAPI:
+        """分库信息"""
         return ShardingAPI(self)
 
     @cached_property
     def presentation(self) -> PresentationAPI:
+        """展示层"""
         return PresentationAPI(self)
 
     @cached_property
     def extra(self) -> ExtraAPI:
+        """其他接口"""
         return ExtraAPI(self)

deepfos-1.1.53/deepfos/api/models/deep_pipeline.py

@@ -0,0 +1,55 @@
+from typing import Any, Optional
+
+from .base import BaseModel
+
+__all__ = [
+    "RunInfo", "FlowInfo",
+]
+
+
+class RunInfo(BaseModel):
+    #: 元素名/编码
+    elementName: str
+    #: 元素类型
+    elementType: str = "DPL"
+    #: 执行参数
+    parameter: Any
+    #: 文件路径
+    path: str = None
+    #: 文件夹id
+    folderId: str = None
+    # 是否在同一个进程执行
+    inProcess: bool = False
+
+
+class Revision(BaseModel):
+    #: 版本号
+    version: str
+    #: 版本名
+    name: str
+
+
+class Configure(BaseModel):
+    #: 超时时间（秒）
+    timeout: int
+    #: 公共脚本
+    prelude: str = None
+    #: 实例名称
+    runNameTemplate: str = None
+    #: 版本
+    revision: Revision
+    #: 状态
+    status: str
+
+
+class FlowInfo(BaseModel):
+    #: 元素名/编码
+    elementName: str
+    #: 元素类型
+    elementType: str
+    #: 文件夹id
+    folderId: str = None
+    #: 元素id
+    elementId: str = None
+    #: 数据流配置
+    configure: Configure

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/api/python.py

@@ -3,8 +3,11 @@ from .models.python import *
 from deepfos.lib.decorator import cached_property
 from typing import List, Dict, Union, Any, Awaitable
 
+__all__ = ['PythonAPI']
+
 
 class WorkerAPI(ChildAPI):
+    """python工作进程相关接口"""
     endpoint = '/worker'
 
     @post('register')
@@ -17,6 +20,7 @@ class WorkerAPI(ChildAPI):
 
 
 class ScriptAPI(ChildAPI):
+    """python脚本的运行，关闭，进度查询等逻辑"""
     endpoint = '/script'
 
     @post('run')
@@ -33,6 +37,7 @@ class ScriptAPI(ChildAPI):
 
 
 class FileAPI(ChildAPI):
+    """python文件的新建，上传，更新逻辑"""
     endpoint = '/file'
 
     @post('add')
@@ -61,6 +66,7 @@ class FileAPI(ChildAPI):
 
 
 class PythonAPI(DynamicRootAPI, builtin=True):
+    """Python组件"""
     module_type = 'PY'
     default_version = (2, 0)
     multi_version = False
@@ -70,15 +76,15 @@ class PythonAPI(DynamicRootAPI, builtin=True):
 
     @cached_property
     def worker(self) -> WorkerAPI:
+        """python工作进程相关接口"""
         return WorkerAPI(self)
 
     @cached_property
     def script(self) -> ScriptAPI:
+        """python脚本的运行，关闭，进度查询等逻辑"""
         return ScriptAPI(self)
 
     @cached_property
     def file(self) -> FileAPI:
-        """
-        python文件管理相关接口
-        """
+        """python文件的新建，上传，更新逻辑"""
         return FileAPI(self)

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/core/logictable/sqlcondition.py

@@ -410,12 +410,20 @@ class SQLCondition:
             val_list = self.val_list.copy()
         elif 1 == len(fd_me) == len(fd_other):
             val_list = pd.DataFrame()
-            self.__static_cond = self.__static_cond.append(
-                pd.DataFrame(data={'field': self.fields[0], 'op': [OpFactory(IN, self.val_list.iloc[:, 0])]})
-            )
-            other.__static_cond = other.__static_cond.append(
-                pd.DataFrame(data={'field': other.fields[0], 'op': [OpFactory(IN, other.val_list.iloc[:, 0])]})
-            )
+            self.__static_cond = pd.concat([
+                self.__static_cond,
+                pd.DataFrame(data={
+                    'field': self.fields[0],
+                    'op': [OpFactory(IN, self.val_list.iloc[:, 0])]
+                })
+            ])
+            other.__static_cond = pd.concat([
+                other.__static_cond,
+                pd.DataFrame(data={
+                    'field': other.fields[0],
+                    'op': [OpFactory(IN, other.val_list.iloc[:, 0])]
+                })
+            ])
         else:
             # 字段越多，限制条件越多，一般数据量越小，定义为sub
             if fd_me.issubset(fd_other):
@@ -465,8 +473,14 @@ class SQLCondition:
 
         if len(fields) == 1:
             # 将条件归类至sta_cond
-            static_cond = self.__static_cond.append(
-                pd.DataFrame(data={'field': fields, 'op': [OpFactory(IN, val_list.iloc[:, 0])]})
+            static_cond = pd.concat(
+                [
+                    self.__static_cond,
+                    pd.DataFrame(data={
+                        'field': fields,
+                        'op': [OpFactory(IN, val_list.iloc[:, 0])]
+                    })
+                ]
             ).sort_values('field').reset_index(drop=True)
             val_list = pd.DataFrame()
         else:

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/core/logictable/tablemodel.py

@@ -339,11 +339,12 @@ class MetaTable(MetaNodeMixin):
 
         df = pd.DataFrame()
         for cd in cond.to_sql():
-            df = df.append(cls.datatable.select(
-                columns=fields,
-                where=cd,
-                distinct=unique,
-            ))
+            df = pd.concat(
+                [
+                    df,
+                    cls.datatable.select(columns=fields, where=cd, distinct=unique)
+                ]
+            )
 
         logger.debug(f"Got DATA:\n {df}")
 

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/element/__init__.py

@@ -35,6 +35,7 @@ if TYPE_CHECKING:  # pragma: no cover
     from .journal import AsyncJournalModel, JournalModel
     from .deepmodel import AsyncDeepModel, DeepModel
     from .deepconnector import AsyncDeepConnector, DeepConnector
+    from .deep_pipeline import AsyncDeepPipeline, DeepPipeline
 
 
 lazify(
@@ -82,6 +83,7 @@ lazify(
         'deepfos.element.journal': ('AsyncJournalModel', 'JournalModel'),
         'deepfos.element.deepmodel': ('AsyncDeepModel', 'DeepModel'),
         'deepfos.element.deepconnector': ('AsyncDeepConnector', 'DeepConnector'),
+        'deepfos.element.deep_pipeline': ('AsyncDeepPipeline', 'DeepPipeline'),
     },
     globals()
 )

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/element/bizmodel.py

@@ -646,7 +646,7 @@ class AsyncBusinessModel(ElementBase[BusinessModelAPI]):
             df_row = pd.DataFrame([{'process_operation_id': operation_info.id,
                                     'origin_status': operation_info.originStatusList,
                                     'target_status': operation_info.targetStatus}])
-            df_operation = df_operation.append(df_row)
+            df_operation = pd.concat([df_operation, df_row])
         df_operation.reset_index(drop=True, inplace=True)
         if len(df_operation) > 1:
             df_operation['target_status_shift'] = df_operation['target_status'].shift(1)
@@ -699,7 +699,7 @@ class AsyncBusinessModel(ElementBase[BusinessModelAPI]):
                 df_success['line_no'] += 1
                 df_success['result_status'] = row['target_status']
                 df_success['process_operation_id'] = row['process_operation_id']
-                df_insert = df_insert.append(df_success)
+                df_insert = pd.concat([df_insert, df_success])
 
             tbl_main = self.logic_tables[main_tbl_name].async_datatable
             if not df_failure.empty:

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/element/datatable.py

@@ -1690,7 +1690,7 @@ class AsyncDirectAccessDataTableMySQL(AsyncDataTableMySQL):
             limit=limit,
             offset=offset
         )
-        return raw_data.to_dict(orient='record')
+        return raw_data.to_dict(orient='records')
 
     @txn_support
     async def run_sql(self, sql: str):

deepfos-1.1.53/deepfos/element/deep_pipeline.py

@@ -0,0 +1,186 @@
+import asyncio
+import time
+from math import ceil
+from typing import Any, TYPE_CHECKING, List
+
+from deepfos import OPTION
+from deepfos.api.deep_pipeline import DeepPipelineAPI
+from deepfos.api.models.deep_pipeline import RunInfo, FlowInfo
+from deepfos.element.base import ElementBase, SyncMeta
+from deepfos.exceptions import (
+    APIResponseError, RunIdInvalid, RunTerminated,
+    ReleaseFlowTimeout, RunFailedError, ReleaseFlowNotExists,
+)
+from deepfos.lib.asynchronous import future_property
+from deepfos.lib.decorator import cached_property
+
+errcode_map = {
+    42030002: ReleaseFlowTimeout,
+    42030003: RunIdInvalid,
+    42030009: RunFailedError,
+    42030010: RunTerminated,
+}
+
+
+# -----------------------------------------------------------------------------
+# core
+class AsyncDeepPipeline(ElementBase[DeepPipelineAPI]):
+    """数据流3.0"""
+
+    @future_property
+    async def release(self) -> FlowInfo:
+        """发布版信息"""
+        api = await self.wait_for('async_api')
+        ele_info = await self.wait_for('element_info')
+        return await api.flow.open(
+            folder_id=ele_info.folderId,
+            element_name=ele_info.elementName,
+            version='release'
+        )
+
+    @cached_property
+    def has_approved_release(self) -> bool:
+        return (
+            self.release is not None
+            and self.release.configure.status == 'APPROVED'
+        )
+
+    async def run(
+        self,
+        parameter: Any = None,
+        timeout: int = None,
+        in_process: bool = True,
+    ) -> Any:
+        """
+        同步启动数据流
+
+        Args:
+            parameter: 执行参数
+            timeout: 超时时间(秒)
+            in_process: 是否在同一个进程执行
+
+        Returns:
+            执行结果
+        """
+        run_id = await self.run_async(parameter, in_process)
+        return await self.result(run_id, timeout)
+
+    async def run_async(
+        self,
+        parameter: Any = None,
+        in_process: bool = True,
+    ) -> str:
+        """
+        异步启动数据流
+
+        Args:
+            parameter: 执行参数
+            in_process: 是否在同一个进程执行
+
+        Returns:
+            执行ID
+        """
+        if not self.has_approved_release:
+            raise ReleaseFlowNotExists('暂无启用中状态的数据流版本')
+
+        return await self.async_api.run.run_async(
+            RunInfo(
+                elementName=self.element_name,
+                parameter=parameter,
+                folderId=self.element_info.folderId,
+                inProcess=in_process
+            )
+        )
+
+    async def result(self, run_id: str, timeout: int = None) -> Any:
+        """
+        获取异步执行结果
+
+        Args:
+            run_id: 执行ID
+            timeout: 超时时间(秒)
+
+        Returns:
+            执行结果
+
+        """
+        start_time = time.time()
+        remaining_time = interval = min(max(OPTION.api.timeout - 1, 1), 5)
+        if timeout is not None:
+            remaining_time = min(interval, timeout)
+
+        while timeout is None or remaining_time > 0:
+            try:
+                return await self.async_api.run.result(
+                    run_id,
+                    timeout=remaining_time
+                )
+            except APIResponseError as e:
+                if e.code not in errcode_map:
+                    raise
+
+                err_cls = errcode_map[e.code]
+                if err_cls is not ReleaseFlowTimeout:
+                    raise errcode_map[e.code](str(e)) from None
+
+                delta_time = time.time() - start_time
+                remaining_time = (
+                    ceil(min(interval, timeout - delta_time))
+                    if timeout is not None else interval
+                )
+
+        raise ReleaseFlowTimeout() from None
+
+    async def run_batch(
+        self,
+        parameters: List[Any],
+        in_process: bool = True,
+    ) -> List[str]:
+        """
+        批量异步启动数据流
+
+        Args:
+            parameters: 执行参数列表
+            in_process: 是否在同一个进程执行
+
+        Returns:
+            执行ID列表
+        """
+        if not self.has_approved_release:
+            raise ReleaseFlowNotExists('暂无启用中状态的数据流版本')
+
+        result = await asyncio.gather(*(
+            self.run_async(parameter, in_process)
+            for parameter in parameters
+        ))
+        return list(result)
+
+
+class DeepPipeline(AsyncDeepPipeline, metaclass=SyncMeta):
+    synchronize = ('run', 'run_async', 'result', 'run_batch')
+
+    if TYPE_CHECKING:  # pragma: no cover
+        def run(
+            self,
+            parameter: Any = None,
+            timeout: int = None,
+            in_process: bool = True,
+        ) -> Any:
+            ...
+
+        def run_async(
+            self,
+            parameter: Any = None,
+            in_process: bool = True,
+        ) -> str:
+            ...
+
+        def result(self, run_id: str, timeout: int = None) -> Any:
+            ...
+
+        def run_batch(
+            self,
+            parameters: List[Any],
+            in_process: bool = True,
+        ):
+            ...

{deepfos-1.1.52 → deepfos-1.1.53}/deepfos/element/dimension.py

@@ -1118,7 +1118,7 @@ class AsyncDimension(ElementBase[DimensionAPI]):
         if 'shared_member' not in dataframe.columns:
             dataframe = dataframe.assign(shared_member=False)
 
-        records = dataframe[fields].to_dict(orient="records")
+        records = dataframe[list(fields)].to_dict(orient="records")
         mbrs = []
         for record in records:
             mbrs.append(
@@ -1173,7 +1173,7 @@ def _validate_df_for_dimension(df: pd.DataFrame):
         duplicated = unique_df[unique_df['size'] > 1]
         if not duplicated.empty:
             raise ValueError(
-                f"Duplicated member name for: {duplicated.to_dict(orient='record')}"
+                f"Duplicated member name for: {duplicated.to_dict(orient='records')}"
             )
     else:
         if df[DFLT_NAME_COLUMN].nunique() != len(df):