deepfos 1.1.54__tar.gz → 1.1.55__tar.gz

{deepfos-1.1.54 → deepfos-1.1.55}/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [1.1.55] - 2025-04-25
+
+### 新增
+* DeepModel支持update_df
+* 财务模型支持complement_save
+
+### 变更
+
+* DeepModel事务内设置globals不生效问题修正
+* DeepModel兼容线上线下模式的json类型数据的处理，保证insert_df和query_df的读写格式一致性
+
+
 ## [1.1.54] - 2025-03-27
 
 ### 变更

{deepfos-1.1.54 → deepfos-1.1.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.54
+Version: 1.1.55
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.54 → deepfos-1.1.55}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-03-27T07:28:11+0000",
+ "date": "2025-04-27T01:54:31+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "a250b021576a02a0e056f950eba25b16a16e9172",
- "version": "1.1.54"
+ "full-revisionid": "8a98c948a65577c31df7cff7c2a512e8d71de104",
+ "version": "1.1.55"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.54 → deepfos-1.1.55}/deepfos/element/deepmodel.py

@@ -128,6 +128,7 @@ class ObjectElement(ObjectParam):
 class QueryWithArgs(BaseModel):
     commands: str
     kwargs: Dict[str, Any] = Field(default_factory=dict)
+    globals: Dict[str, Any] = Field(default_factory=dict)
 
 
 class MainField(NamedTuple):
@@ -370,11 +371,7 @@ class FieldJson(BaseField):
             df[field_name] = df[field_name].apply(self.format_json)
 
     def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
-        # std::json needed to be cast only when data is from http
-        # since json value will be converted to json string(type: str)
-        # in edgedb python protocol
-        if not direct_access:
-            df[field_name] = df[field_name].apply(self.format_json)
+        pass
 
 
 class FieldInt(BaseField):
@@ -488,6 +485,9 @@ def _iter_single_assign(
     assign_string = f"{field.name} := "
     # 设置标量值
     if field.name not in target_main_field:
+        if field.is_multi:
+            return assign_string + f"json_array_unpack(item['{field.name}'])"
+
         assign_string += f"<{cast_type}>"
 
         if cast_type in NEED_CAST_STR:
@@ -590,10 +590,10 @@ def bulk_upsert_by_fields(
 
 def bulk_update_by_fields(
     object_name: str,
-    business_key: str,
     field_type: List[PtrInfo],
     target_main_field: Dict[str, MainField],
-    update_fields: Iterable[str] = None,
+    match_fields: Iterable[str],
+    update_fields: Iterable[str],
 ):
     update_assign_body = ','.join(
         [
@@ -602,11 +602,20 @@ def bulk_update_by_fields(
         ]
     )
 
+    field_type_map = {field.name: field.type for field in field_type}
+
+    match_str = " and ".join(
+        [
+            f".{name} = <{field_type_map.get(name, 'std::str')}>item['{name}']"
+            for name in match_fields
+        ]
+    )
+
     return f"""
         with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
         for item in json_array_unpack(raw_data) union (
             update {object_name} 
-            filter .{business_key} = <std::str>item['{business_key}']
+            filter {match_str}
             set {{
                 {update_assign_body}
             }}
@@ -777,6 +786,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         self.direct_access = direct_access
         self.alias = AliasGenerator()
         self.pg_dsn = pg_dsn
+        self._globals = {}
 
     @future_property
     async def client(self):
@@ -799,12 +809,12 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                 dbname=dbname
             )
             if user_id := OPTION.api.header.get('user'):
-                client = client.with_globals(
-                    **{
-                        f'{self.spacemodule}::current_user_id':
-                        user_id
-                    }
-                )
+                default_globals = {
+                    f'{self.spacemodule}::current_user_id':
+                    user_id
+                }
+                client = client.with_globals(**default_globals)
+                self._globals = client._options.state._globals
             return client
 
     @future_property
@@ -899,12 +909,18 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         """
 
         if self.direct_access:
-            logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
+            logger.opt(lazy=True).debug(f"Query: [{ql}],\n"
+                                        f"kwargs: [{kwargs}],\n"
+                                        f"globals: [{self._globals}].")
             _, result = await self.client.query(ql, **kwargs)
             return result
 
-        result = await AsyncDeepModel.query(self, ql, **kwargs)
-        return serutils.deserialize(result)
+        result = await self._http_query(ql, **kwargs)
+        field_info = {
+            fi.name: fi.type if fi.fields is None else fi.fields
+            for fi in result.objectInfos[0].fields
+        } if result.objectInfos else {}
+        return serutils.deserialize(result.json_, field_info)
 
     async def query(self, ql: str, **kwargs) -> List[Any]:
         """执行ql查询语句，得到序列化后的结果
@@ -934,7 +950,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
 
         """
         if self.direct_access:
-            logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
+            logger.opt(lazy=True).debug(f"Query: [{ql}],\n"
+                                        f"kwargs: [{kwargs}],\n"
+                                        f"globals: [{self._globals}].")
             frame_desc, result = await self.client.query(ql, **kwargs)
             return serutils.serialize(
                 result, ctx=serutils.Context(frame_desc=frame_desc)
@@ -995,7 +1013,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         else:
             result = await self._http_query(ql, **kwargs)
             # No object structure info
-            if result.objectInfos is None:
+            if not result.objectInfos:
                 return pd.DataFrame(data=result.json_)
 
             data = pd.DataFrame(
@@ -1015,6 +1033,10 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
     query_object.__doc__ = query_object.__doc__ + DOC_ARGS_KWARGS
     query_df.__doc__ = query_df.__doc__ + DOC_ARGS_KWARGS
 
+    def _ensure_client(self):
+        if self.direct_access:
+            self.client # noqa
+
     @txn_support
     async def execute(
         self,
@@ -1030,8 +1052,11 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                  会自动用作所有string形式ql的参数
 
         """
+        self._ensure_client()
         if isinstance(qls, str):
-            qls_with_args = [QueryWithArgs(commands=qls, kwargs=kwargs)]
+            qls_with_args = [QueryWithArgs(
+                commands=qls, kwargs=kwargs, globals=self._globals
+            )]
         else:
             qls_with_args = []
             seen_kwargs_key = set()
@@ -1049,7 +1074,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                         seen_kwargs_key = seen_kwargs_key.union(ql.kwargs.keys())
 
                 elif isinstance(ql, str):
-                    qls_with_args.append(QueryWithArgs(commands=ql, kwargs=kwargs))
+                    qls_with_args.append(QueryWithArgs(
+                        commands=ql, kwargs=kwargs, globals=self._globals
+                    ))
                 else:
                     raise TypeError(f'qls参数中出现类型非法成员：{type(ql)}')
 
@@ -1089,12 +1116,25 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             async with tx:
                 for ql in qls_with_args:
                     logger.opt(lazy=True).debug(
-                        f"Execute QL: [{ql.commands}], \nkwargs: [{ql.kwargs}]."
+                        f"Execute QL: [{ql.commands}],"
+                        f"\nkwargs: [{ql.kwargs}],"
+                        f"\nglobals: [{ql.globals}]."
                     )
-                    desc, affected = await tx.execute(ql.commands, **ql.kwargs)
-                    result.append(serutils.serialize(
-                        affected, ctx=serutils.Context(frame_desc=desc)
-                    ))
+                    if ql.globals:
+                        bak_cli = tx._client
+                        tx._client = tx._client.with_globals(**ql.globals)
+                        try:
+                            desc, affected = await tx.execute(ql.commands, **ql.kwargs)
+                            result.append(serutils.serialize(
+                                affected, ctx=serutils.Context(frame_desc=desc)
+                            ))
+                        finally:
+                            tx._client = bak_cli
+                    else:
+                        desc, affected = await tx.execute(ql.commands, **ql.kwargs)
+                        result.append(serutils.serialize(
+                            affected, ctx=serutils.Context(frame_desc=desc)
+                        ))
         if len(result) == 1:
             return result[0]
         return result
@@ -1176,6 +1216,128 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             if exclusive not in valid_exclusive:
                 raise ValueError(f"exclusive fields: {exclusive_fields} 没有相应唯一约束")
 
+    async def _get_bkey(
+        self,
+        obj: Union[ObjectTypeFrame, TargetField],
+        source: str = None,
+        name: str = None
+    ) -> str:
+        # 如可在object结构的annotations中取业务主键，则优先取，否则走接口
+        if obj.info and BUSINESS_KEY in obj.info:
+            return obj.info[BUSINESS_KEY]
+        elif (code := obj.normalized_name) in self.model_objects:
+            return self.model_objects[code].businessKey
+
+        assert isinstance(obj, TargetField)
+        # Link 至非本应用对象，需单独查询
+        tgt = ObjectElement.construct_from(self.model_objects[source]).links[name]
+        tgt_model_info = await self.async_api.object.info(
+            app=tgt.targetApp, object_code=tgt.targetObjectCode
+        )
+        return tgt_model_info.businessKey
+
+    async def _collect_bulk_field_info(self, object_name, structure, data, relation):
+        field_info = []
+        tgt_main_field = {}
+        for field in structure.fields.values():
+            if field.name not in data.columns:
+                continue
+
+            field_info.append(field)
+
+            if not field.is_link:
+                continue
+
+            is_multi = field.is_multi_link
+            name = field.name
+            # 链接至其他对象，记录目标对象信息
+            if is_multi:
+                link_props = set(relation[name].columns).intersection(field.props)
+            else:
+                link_props = set(
+                    c[len(f'{name}@')::]
+                    for c in data.columns if c.startswith(f'{name}@')
+                ).intersection(field.props)
+            tgt_bkey = await self._get_bkey(field.target, object_name, name)
+            tgt_main_field[name] = MainField(tgt_bkey, is_multi, link_props)
+        return field_info, tgt_main_field
+
+    def _collect_qls(
+        self,
+        data: pd.DataFrame,
+        ql: str,
+        chunksize: int,
+        qls: List[QueryWithArgs]
+    ):
+        self._ensure_client()
+        for i in range(0, len(data), chunksize):
+            part = data.iloc[i: i + chunksize]
+            kw_name = self.alias.get(BATCH_INSERT_KW)
+            qls.append(QueryWithArgs(
+                commands=ql.replace(
+                    f'${BATCH_INSERT_KW}', f'${kw_name}'
+                ),
+                kwargs={kw_name: part.to_json(
+                    orient='records', double_precision=15,
+                    force_ascii=False, default_handler=str
+                )},
+                globals=self._globals
+            ))
+
+    @staticmethod
+    def _split_self_link(data, relation, structure, bkey):
+        self_link_dfs = {}
+        for name in structure.self_link_fields:
+            field = structure.fields[name]
+            if (link_df := relation.get(name)) is not None:
+                link_props = set(link_df.columns).intersection(field.props)
+                self_link_dfs[name] = (
+                    structure.fit(data[[bkey, name]]),
+                    MainField(bkey, field.is_multi_link, link_props)
+                )
+                data = data.drop(columns=[name])
+            elif name in data.columns:
+                link_prop_cols = []
+                link_props = []
+
+                for col in data.columns:
+                    if (
+                        col.startswith(f'{name}@')
+                        and ((prop_name := col[len(f'{name}@')::]) in field.props)
+                    ):
+                        link_prop_cols.append(col)
+                        link_props.append(prop_name)
+
+                self_link_dfs[name] = (
+                    structure.fit(data[[bkey, name, *link_prop_cols]]),
+                    MainField(bkey, field.is_multi_link, link_props)
+                )
+                data = data.drop(columns=[name, *link_prop_cols])
+        return data, self_link_dfs
+
+    @staticmethod
+    def _merge_relation(data, relation, structure, bkey):
+        for name, link_df in relation.items():
+            if name not in structure.fields:
+                continue
+            field = structure.fields[name]
+            valid_cols = list({'source', 'target', *field.props} & set(link_df.columns))
+            link_df = link_df[valid_cols]
+            # for fit only
+            temp_structure = ObjectStructure(
+                field.type,
+                [
+                    PtrInfo(name='source', target=TargetField(name='std::str')),
+                    PtrInfo(name='target', target=TargetField(name='std::str')),
+                    *[PtrInfo(**prop.dict()) for prop in field.properties]
+                ]
+            )
+            link_df = temp_structure.fit(link_df)
+            link = link_df.groupby('source').apply(_format_link, link_name=name)
+            data = data.drop(columns=[name], errors='ignore')
+            data = data.join(link.to_frame(name), on=bkey)
+        return data
+
     @txn_support
     async def insert_df(
         self,
@@ -1297,38 +1459,15 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         data = self._merge_relation(data, relation, structure, bkey)
         # 从data中分离出self-link更新信息
         data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
-
-        field_info = []
-        tgt_main_field = {}
-        # 准备bulk insert所需field信息
-        for field in structure.fields.values():
-            if field.name not in data.columns:
-                continue
-
-            field_info.append(field)
-
-            if not field.is_link:
-                continue
-
-            is_multi = field.is_multi_link
-            name = field.name
-            # 链接至其他对象，记录目标对象信息
-            if is_multi:
-                link_props = set(relation[name].columns).intersection(field.props)
-            else:
-                link_props = set(
-                    c[len(f'{name}@')::]
-                    for c in data.columns if c.startswith(f'{name}@')
-                ).intersection(field.props)
-            tgt_bkey = await self._get_bkey(field.target, object_name, name)
-            tgt_main_field[name] = MainField(tgt_bkey, is_multi, link_props)
-
+        field_info, tgt_main_field = await self._collect_bulk_field_info(
+            object_name, structure, data, relation
+        )
         field_names = set(map(lambda f: f.name, field_info))
         if enable_upsert:
             self._valid_upsert(obj, field_names, bkey, exclusive_fields, update_fields)
 
         exclusive_fields = set(exclusive_fields or {bkey}) & set(field_names)
-        update_fields = set(update_fields or (field_names- {bkey})) & set(field_names)
+        update_fields = set(update_fields or (field_names - {bkey})) & set(field_names)
         if enable_upsert and update_fields:
             insert_ql = bulk_upsert_by_fields(
                 object_name, field_info, tgt_main_field,
@@ -1343,30 +1482,13 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             for update_field, (update_df, main_field) in self_link_dfs.items():
                 field = structure.fields[update_field]
                 update_ql = bulk_update_by_fields(
-                    object_name, bkey, [field],
-                    {update_field: main_field}, [update_field]
+                    object_name, [field], {update_field: main_field},
+                    [bkey], [update_field]
                 )
                 self._collect_qls(update_df, update_ql, chunksize, qls)
 
         await self.execute(qls)
 
-    def _collect_qls(
-        self,
-        data: pd.DataFrame,
-        ql: str,
-        chunksize: int,
-        qls: List[QueryWithArgs]
-    ):
-        for i in range(0, len(data), chunksize):
-            part = data.iloc[i: i + chunksize]
-            kw_name = self.alias.get(BATCH_INSERT_KW)
-            qls.append(QueryWithArgs(
-                commands=ql.replace(
-                    f'${BATCH_INSERT_KW}', f'${kw_name}'
-                ),
-                kwargs={kw_name: part.to_json(orient='records', double_precision=15)}
-            ))
-
     async def get_object(
         self,
         object_name: str,
@@ -1395,80 +1517,6 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
 
         return format_obj(objs[0])
 
-    async def _get_bkey(
-        self,
-        obj: Union[ObjectTypeFrame, TargetField],
-        source: str = None,
-        name: str = None
-    ) -> str:
-        # 如可在object结构的annotations中取业务主键，则优先取，否则走接口
-        if obj.info and BUSINESS_KEY in obj.info:
-            return obj.info[BUSINESS_KEY]
-        elif (code := obj.normalized_name) in self.model_objects:
-            return self.model_objects[code].businessKey
-
-        assert isinstance(obj, TargetField)
-        # Link 至非本应用对象，需单独查询
-        tgt = ObjectElement.construct_from(self.model_objects[source]).links[name]
-        tgt_model_info = await self.async_api.object.info(
-            app=tgt.targetApp, object_code=tgt.targetObjectCode
-        )
-        return tgt_model_info.businessKey
-
-    @staticmethod
-    def _split_self_link(data, relation, structure, bkey):
-        self_link_dfs = {}
-        for name in structure.self_link_fields:
-            field = structure.fields[name]
-            if (link_df := relation.get(name)) is not None:
-                link_props = set(link_df.columns).intersection(field.props)
-                self_link_dfs[name] = (
-                    structure.fit(data[[bkey, name]]),
-                    MainField(bkey, field.is_multi_link, link_props)
-                )
-                data = data.drop(columns=[name])
-            elif name in data.columns:
-                link_prop_cols = []
-                link_props = []
-
-                for col in data.columns:
-                    if (
-                        col.startswith(f'{name}@')
-                        and ((prop_name := col[len(f'{name}@')::]) in field.props)
-                    ):
-                        link_prop_cols.append(col)
-                        link_props.append(prop_name)
-
-                self_link_dfs[name] = (
-                    structure.fit(data[[bkey, name, *link_prop_cols]]),
-                    MainField(bkey, field.is_multi_link, link_props)
-                )
-                data = data.drop(columns=[name, *link_prop_cols])
-        return data, self_link_dfs
-
-    @staticmethod
-    def _merge_relation(data, relation, structure, bkey):
-        for name, link_df in relation.items():
-            if name not in structure.fields:
-                continue
-            field = structure.fields[name]
-            valid_cols = list({'source', 'target', *field.props} & set(link_df.columns))
-            link_df = link_df[valid_cols]
-            # for fit only
-            temp_structure = ObjectStructure(
-                field.type,
-                [
-                    PtrInfo(name='source', target=TargetField(name='std::str')),
-                    PtrInfo(name='target', target=TargetField(name='std::str')),
-                    *[PtrInfo(**prop.dict()) for prop in field.properties]
-                ]
-            )
-            link_df = temp_structure.fit(link_df)
-            link = link_df.groupby('source').apply(_format_link, link_name=name)
-            data = data.drop(columns=[name], errors='ignore')
-            data = data.join(link.to_frame(name), on=bkey)
-        return data
-
     async def insert_df_pg(
         self,
         object_name: str,
@@ -1584,6 +1632,80 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         finally:
             await conn.close()
 
+    @txn_support
+    async def update_df(
+        self,
+        object_name: str,
+        data: pd.DataFrame,
+        relation: Dict[str, pd.DataFrame] = None,
+        chunksize: int = 500,
+        match_fields: Iterable[str] = None,
+    ) -> None:
+        """以事务执行基于DataFrame字段信息的批量更新数据
+
+        将以业务主键作为匹配条件，除业务主键以外的字段将为update fields
+
+        Args:
+            object_name: 被插入数据的对象名，需属于当前应用
+            data: 要插入的数据，若有single link property，
+                  则以列名为link_name@link_property_name的形式提供
+            relation: 如有multi link，提供该字典用于补充link target信息，
+                    键为link字段名，值为映射关系的DataFrame
+                    DataFrame中的source列需为插入对象的业务主键，
+                    target列需为link target的业务主键，
+                    若有link property，则以property名为列名，提供在除source和target的列中
+            chunksize: 单次最大行数
+            match_fields: update的匹配列表，涉及的fields需出现在data或relation中，默认为业务主键
+        """
+        if data.empty:
+            logger.info("data为空，无DML执行")
+            return
+
+        if object_name in self.objects:
+            obj = self.objects[object_name]
+        else:
+            raise ObjectNotExist(
+                f'DeepModel对象[{object_name}]在当前应用不存在，无法更新数据'
+            )
+        if obj.external:
+            raise ExternalObjectReadOnly('外部对象只可读')
+
+        structure = ObjectStructure(name=obj.name, structure=obj.fields.values())
+        self._valid_data(data, object_name, relation, structure)
+        relation = relation or {}
+        bkey = await self._get_bkey(obj)
+        # data拼接relation df
+        data = self._merge_relation(data, relation, structure, bkey)
+        # 从data中分离出self-link更新信息
+        data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
+        field_info, tgt_main_field = await self._collect_bulk_field_info(
+            object_name, structure, data, relation
+        )
+        field_names = set(map(lambda f: f.name, field_info))
+
+        if missing := (set(match_fields) - set(field_names)):
+            raise ValueError(f"match fields: {missing} 不在提供的数据中")
+
+        match_fields = set(match_fields or {bkey}) & set(field_names)
+        update_ql = bulk_update_by_fields(
+            object_name, field_info, tgt_main_field,
+            match_fields, field_names - match_fields
+        )
+        qls = []
+        self._collect_qls(structure.fit(data), update_ql, chunksize, qls)
+        if self_link_dfs:
+            for update_field, (update_df, main_field) in self_link_dfs.items():
+                field = structure.fields[update_field]
+                update_ql = bulk_update_by_fields(
+                    object_name, [field],
+                    {update_field: main_field},
+                    [bkey],
+                    [update_field]
+                )
+                self._collect_qls(update_df, update_ql, chunksize, qls)
+
+        await self.execute(qls)
+
     @asynccontextmanager
     async def start_transaction(self, flatten: bool = False):
         """开启事务
@@ -1661,11 +1783,14 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                 raise NotImplemented('非直连模式不支持设置state信息')
         else:
             bak_cli = self.client
+            bak_globals = self.client._options.state._globals
             try:
                 self.client = self.client.with_globals(**globals_)
+                self._globals = self.client._options.state._globals
                 yield
             finally:
                 self.client = bak_cli
+                self._globals = bak_globals
 
     @contextmanager
     def without_globals(self, *global_names):
@@ -1676,18 +1801,22 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                 raise NotImplemented('非直连模式不支持设置state信息')
         else:
             bak_cli = self.client
+            bak_globals = self.client._options.state._globals
             try:
                 self.client = self.client.without_globals(*global_names)
+                self._globals = self.client._options.state._globals
                 yield
             finally:
                 self.client = bak_cli
+                self._globals = bak_globals
 
 
 class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
     synchronize = (
         'query_object', 'query', 'query_df',
         'execute', 'get_object',
-        'insert_df', 'insert_df_pg'
+        'insert_df', 'insert_df_pg',
+        'update_df',
     )
 
     if TYPE_CHECKING:  # pragma: no cover
@@ -1737,6 +1866,16 @@ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
         ) -> None:
             ...
 
+        def update_df(
+            self,
+            object_name: str,
+            data: pd.DataFrame,
+            relation: Dict[str, pd.DataFrame] = None,
+            chunksize: int = 500,
+            match_fields: Iterable[str] = None,
+        ) -> None:
+            ...
+
     @contextmanager
     def start_transaction(self, flatten: bool = False):
         """开启事务