deepfos 1.1.66__tar.gz → 1.1.68__tar.gz

{deepfos-1.1.66 → deepfos-1.1.68}/CHANGELOG.md

@@ -1,3 +1,24 @@
+## [1.1.68] - 2025-07-29
+
+### 更新
+
+* DeepModel元素insert_df和update_df接受chunksize=None
+* DeepModel元素insert_df和update_df组织的bulk语句优化
+* 修复DataFrame.replace会引发类型推断导致的bug
+
+
+## [1.1.67] - 2025-07-22
+
+### 新增
+
+* 财务模型元素查询支持指定透视成员列表
+* 凭证模型数据更新接受指定头行操作类型
+
+### 更新
+
+* 兼容维度load_dataframe依据共享成员列分数据时共享成员列非bool列的处理
+
+
 ## [1.1.66] - 2025-07-10
 
 ### 更新

{deepfos-1.1.66 → deepfos-1.1.68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.66
+Version: 1.1.68
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.66 → deepfos-1.1.68}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-07-10T07:24:49+0000",
+ "date": "2025-08-07T03:10:50+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "7742bce18499f0a93f0279f4e2d18c6a9113f288",
- "version": "1.1.66"
+ "full-revisionid": "e1b02f36c62e37f03965c5219dc9426d19af95a8",
+ "version": "1.1.68"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.66 → deepfos-1.1.68}/deepfos/element/deepmodel.py

@@ -1,4 +1,5 @@
 import re
+import textwrap
 import threading
 
 import numpy as np
@@ -7,7 +8,7 @@ import json
 import uuid
 from contextlib import asynccontextmanager, contextmanager
 from contextvars import ContextVar
-from itertools import count
+from itertools import count, chain
 from typing import (
     List, TYPE_CHECKING, Any, Dict, Union, NamedTuple,
     Iterable, Optional, Literal
@@ -37,7 +38,7 @@ from deepfos.lib import serutils
 from deepfos.lib.asynchronous import future_property, evloop
 from deepfos.lib.decorator import flagmethod, cached_property, lru_cache
 from deepfos.lib.utils import (
-    AliasGenerator, to_version_tuple, ChunkAlert, split_dataframe_alert
+    AliasGenerator, to_version_tuple,
 )
 
 __all__ = ['AsyncDeepModel', 'DeepModel', 'to_fields', 'QueryWithArgs']
@@ -121,6 +122,7 @@ dm_type_to_edb_scalar = {
     'uuid': 'std::str',
     'json': 'std::json',
 }
+TAB = ' ' * 4
 
 
 class ObjectElement(ObjectParam):
@@ -485,19 +487,30 @@ def _iter_single_assign(
     field: PtrInfo,
     cast_type: str,
     target_main_field: Dict[str, MainField]
-):
-    assign_string = f"{field.name} := "
+) -> str:
+    """
+    生成单字段赋值语句
+
+    Args:
+        field: 字段信息
+        cast_type: 字段类型
+        target_main_field: 目标字段信息
+
+    Returns:
+        赋值语句
+    """
+    assign = f"\n{field.name} := "
     # 设置标量值
     if field.name not in target_main_field:
         if field.is_multi:
-            return assign_string + f"json_array_unpack(item['{field.name}'])"
+            return assign + f"json_array_unpack(item['{field.name}'])"
 
-        assign_string += f"<{cast_type}>"
+        assign += f"<{cast_type}>"
 
         if cast_type in NEED_CAST_STR:
-            assign_string += '<std::str>'
+            assign += '<std::str>'
 
-        return assign_string + f"item['{field.name}']"
+        return assign + f"item['{field.name}']"
 
     # 设置link target值
     link = field.name
@@ -516,20 +529,20 @@ def _iter_single_assign(
         target = cast_type
 
     if main_field.is_multi:
-        assign_string += f"""(
-            select detached {target}
-            filter contains(
-                <array<{main_field.type}>>(json_get(item, '{link}', 'target')),
-                .{main_field.business_key}
-            )
-        )"""
+        assign += 'distinct (\n' + textwrap.indent(textwrap.dedent(f"""\
+        for each_{link} in json_array_unpack(json_get(item, '{link}', 'target'))
+        union (
+            select detached {target} 
+            filter .{main_field.business_key} = <{main_field.type}>each_{link}
+        )"""), TAB) + '\n)'
     else:
-        assign_string += f"""assert_single((
+        assign += textwrap.dedent(f"""\
+        assert_single((
             select detached {target}
             filter .{main_field.business_key} = <{main_field.type}>(json_get(item, '{link}'))
-        ))"""
+        ))""")
 
-    return assign_string
+    return assign
 
 
 def bulk_insert_by_fields(
@@ -537,21 +550,16 @@ def bulk_insert_by_fields(
     field_type: List[PtrInfo],
     target_main_field: Dict[str, MainField],
 ):
-    insert_assign_body = ','.join(
-        [
-            _iter_single_assign(field, field.type, target_main_field)
-            for field in field_type
-        ]
-    )
-
-    return f"""
-    with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
-    for item in json_array_unpack(raw_data) union (
-        insert {object_name} {{
-            {insert_assign_body}
-        }}
-    )
-    """
+    insert_assign_body = ','.join([
+        _iter_single_assign(field, field.type, target_main_field)
+        for field in field_type
+    ])
+    return textwrap.dedent(f"""
+        with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
+        for item in json_array_unpack(raw_data) union (
+            insert {object_name} {{{textwrap.indent(insert_assign_body, TAB * 4)}
+            }}
+        )""")
 
 
 def bulk_upsert_by_fields(
@@ -562,34 +570,27 @@ def bulk_upsert_by_fields(
     update_fields: Iterable[str]
 ):
     conflict_on_fields = map(lambda n: f'.{n}', exclusive_fields)
-
-    insert_assign_body = ','.join(
-        [
-            _iter_single_assign(field, field.type, target_main_field)
-            for field in field_type
-        ]
-    )
+    insert_assign_body = ','.join([
+        _iter_single_assign(field, field.type, target_main_field)
+        for field in field_type
+    ])
     update_assign_body = ','.join(
         [
             _iter_single_assign(field, field.type, target_main_field)
             for field in field_type if field.name in update_fields
         ]
     )
-
-    return f"""
+    return textwrap.dedent(f"""
         with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
         for item in json_array_unpack(raw_data) union (
-            insert {object_name} {{
-                {insert_assign_body}
+            insert {object_name} {{{textwrap.indent(insert_assign_body, TAB * 4)}
             }}
             unless conflict on ({','.join(conflict_on_fields)})
             else (
-                update {object_name} set {{
-                    {update_assign_body}
+                update {object_name} set {{{textwrap.indent(update_assign_body, TAB * 5)}
                 }}
             )
-        )
-        """
+        )""")
 
 
 def bulk_update_by_fields(
@@ -599,32 +600,27 @@ def bulk_update_by_fields(
     match_fields: Iterable[str],
     update_fields: Iterable[str],
 ):
-    update_assign_body = ','.join(
-        [
-            _iter_single_assign(field, field.type, target_main_field)
-            for field in field_type if field.name in update_fields
-        ]
-    )
+    update_assign_body = ','.join([
+        _iter_single_assign(field, field.type, target_main_field)
+        for field in field_type if field.name in update_fields
+    ])
 
     field_type_map = {field.name: field.type for field in field_type}
-
     match_str = " and ".join(
         [
             f".{name} = <{field_type_map.get(name, 'std::str')}>item['{name}']"
             for name in match_fields
         ]
     )
-
-    return f"""
-        with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
-        for item in json_array_unpack(raw_data) union (
-            update {object_name} 
-            filter {match_str}
-            set {{
-                {update_assign_body}
-            }}
-        )
-        """
+    return textwrap.dedent(f"""
+    with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
+    for item in json_array_unpack(raw_data) union (
+        update {object_name} 
+        filter {match_str}
+        set {{{textwrap.indent(update_assign_body, TAB * 3)}
+        }}
+    )
+    """)
 
 
 def format_obj(obj: edgedb.Object) -> ObjectTypeFrame:
@@ -793,8 +789,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         self,
         direct_access: bool = True,
         pg_dsn: str = None,
-        before_chunk: ChunkAlert = None,
-        after_chunk: ChunkAlert = None,
+        **kwargs
     ):
         self._txn_ = ContextVar('QLTXN')
         self.appmodule = f"app{OPTION.api.header['app']}"
@@ -805,8 +800,6 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         self.alias = AliasGenerator()
         self.pg_dsn = pg_dsn
         self._globals = None
-        self.before_chunk = before_chunk
-        self.after_chunk = after_chunk
         self._clients = threading.local()
 
     @future_property
@@ -1099,6 +1092,10 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                  会自动用作所有string形式ql的参数
 
         """
+        qls_with_args = self._collect_execute_qls(qls, kwargs)
+        return await self._maybe_exec_qls(qls_with_args)
+
+    def _collect_execute_qls(self, qls, kwargs):
         self._ensure_client()
         if isinstance(qls, str):
             qls_with_args = [QueryWithArgs(
@@ -1126,8 +1123,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                     ))
                 else:
                     raise TypeError(f'qls参数中出现类型非法成员：{type(ql)}')
-
-        return await self._maybe_exec_qls(qls_with_args)
+        return qls_with_args
 
     execute.__doc__ = execute.__doc__ + DOC_ARGS_KWARGS
 
@@ -1302,6 +1298,10 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             name = field.name
             # 链接至其他对象，记录目标对象信息
             if is_multi:
+                if name not in relation:
+                    raise ValueError(
+                        f'对象[{object_name}]的多选链接:[{name}]未定义在relation中'
+                    )
                 link_props = set(relation[name].columns).intersection(field.props)
             else:
                 link_props = set(
@@ -1312,30 +1312,17 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             tgt_main_field[name] = MainField(tgt_bkey, is_multi, link_props)
         return field_info, tgt_main_field
 
-    def _collect_qls(
-        self,
-        data: pd.DataFrame,
-        ql: str,
-        chunksize: int,
-        qls: List[QueryWithArgs]
-    ):
+    def _ql_payload(self, data: pd.DataFrame, ql: str,):
         self._ensure_client()
-
-        for part, alert in split_dataframe_alert(
-            data, chunksize, self.before_chunk, self.after_chunk
-        ):
-            with alert:
-                kw_name = self.alias.get(BATCH_INSERT_KW)
-                qls.append(QueryWithArgs(
-                    commands=ql.replace(
-                        f'${BATCH_INSERT_KW}', f'${kw_name}'
-                    ),
-                    kwargs={kw_name: part.to_json(
-                        orient='records', double_precision=15,
-                        force_ascii=False, default_handler=str
-                    )},
-                    globals=self._globals
-                ))
+        kw_name = self.alias.get(BATCH_INSERT_KW)
+        return QueryWithArgs(
+            commands=ql.replace(f'${BATCH_INSERT_KW}', f'${kw_name}'),
+            kwargs={kw_name: data.to_json(
+                orient='records', double_precision=15,
+                force_ascii=False, default_handler=str
+            )},
+            globals=self._globals,
+        )
 
     @staticmethod
     def _split_self_link(data, relation, structure, bkey):
@@ -1391,6 +1378,89 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             data = data.join(link.to_frame(name), on=bkey)
         return data
 
+    async def _collect_bulk_qls(
+        self,
+        object_name: str,
+        data: pd.DataFrame,
+        relation: Dict[str, pd.DataFrame] = None,
+        chunk_size: int = 500,
+        enable_upsert: bool = False,
+        update_fields: Iterable[str] = None,
+        exclusive_fields: Iterable[str] = None,
+        match_fields: Iterable[str] = None,
+        insert: bool = True
+    ) -> List[List[QueryWithArgs]]:
+        if object_name in self.objects:
+            obj = self.objects[object_name]
+        else:
+            raise ObjectNotExist(
+                f'DeepModel对象[{object_name}]在当前应用不存在，无法插入数据'
+            )
+        if obj.external:
+            raise ExternalObjectReadOnly('外部对象只可读')
+
+        structure = ObjectStructure(name=obj.name, structure=obj.fields.values())
+        self._valid_data(data, object_name, relation, structure, check_required=insert)
+
+        relation = relation or {}
+        bkey = await self._get_bkey(obj)
+        if bkey not in data.columns:
+            raise RequiredFieldUnfilled(f'缺少业务主键[{bkey}]')
+
+        # data拼接relation df
+        data = self._merge_relation(data, relation, structure, bkey)
+        # 从data中分离出self-link更新信息
+        data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
+        field_info, tgt_main_field = await self._collect_bulk_field_info(
+            object_name, structure, data, relation
+        )
+        field_names = set(map(lambda f: f.name, field_info))
+        if insert:
+            if enable_upsert:
+                self._valid_upsert(obj, field_names, bkey, exclusive_fields, update_fields)
+
+            exclusive_fields = set(exclusive_fields or {bkey}) & set(field_names)
+            update_fields = set(update_fields or (field_names - {bkey})) & set(field_names)
+            if enable_upsert and update_fields:
+                bulk_ql = bulk_upsert_by_fields(
+                    object_name, field_info, tgt_main_field,
+                    exclusive_fields, update_fields
+                )
+            else:
+                bulk_ql = bulk_insert_by_fields(object_name, field_info, tgt_main_field)
+        else:
+            if missing := (set(match_fields or [bkey]) - set(field_names)):
+                raise ValueError(f"match fields: {missing} 不在提供的数据中")
+
+            match_fields = set(match_fields or [bkey]) & set(field_names)
+            if to_upd := (field_names - match_fields):
+                bulk_ql = bulk_update_by_fields(
+                    object_name, field_info, tgt_main_field,
+                    match_fields, to_upd
+                )
+            else:
+                bulk_ql = None
+        qls = []
+        self._ensure_client()
+        if chunk_size is None:
+            chunk_size = len(data)
+        for i in range(0, len(data), chunk_size):
+            part = structure.fit(data.iloc[i: i + chunk_size])
+            ql_chunk = []
+            # Ignore bulk_ql when only update multi links
+            if bulk_ql is not None:
+                ql_chunk = [self._ql_payload(part, bulk_ql)]
+            for update_field, (update_df, main_field) in self_link_dfs.items():
+                field = structure.fields[update_field]
+                update_ql = bulk_update_by_fields(
+                    object_name, [field], {update_field: main_field},
+                    [bkey], [update_field]
+                )
+                update_part = update_df.iloc[i: i + chunk_size]
+                ql_chunk.append(self._ql_payload(update_part, update_ql))
+            qls.append(ql_chunk)
+        return qls
+
     @txn_support
     async def insert_df(
         self,
@@ -1401,6 +1471,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         enable_upsert: bool = False,
         update_fields: Iterable[str] = None,
         exclusive_fields: Iterable[str] = None,
+        commit_per_chunk: bool = False,
     ) -> None:
         """以事务执行基于DataFrame字段信息的批量插入数据
 
@@ -1421,6 +1492,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             exclusive_fields: upsert句式下update的exclusive fields列表，
                             涉及的fields需出现在data或relation中，
                             默认为业务主键
+            commit_per_chunk: 每次插入后是否提交事务，
+                            默认为False，即所有数据插入后再提交事务
+                            该参数仅在非start transaction上下文中生效
 
         Notes:
 
@@ -1493,54 +1567,16 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             logger.info("data为空，无DML执行")
             return
 
-        if object_name in self.objects:
-            obj = self.objects[object_name]
-        else:
-            raise ObjectNotExist(
-                f'DeepModel对象[{object_name}]在当前应用不存在，无法插入数据'
-            )
-        if obj.external:
-            raise ExternalObjectReadOnly('外部对象只可读')
-
-        structure = ObjectStructure(name=obj.name, structure=obj.fields.values())
-
-        self._valid_data(data, object_name, relation, structure)
-
-        relation = relation or {}
-        bkey = await self._get_bkey(obj)
-        # data拼接relation df
-        data = self._merge_relation(data, relation, structure, bkey)
-        # 从data中分离出self-link更新信息
-        data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
-        field_info, tgt_main_field = await self._collect_bulk_field_info(
-            object_name, structure, data, relation
+        qls = await self._collect_bulk_qls(
+            object_name, data, relation, chunksize,
+            enable_upsert, update_fields, exclusive_fields,
+            insert=True
         )
-        field_names = set(map(lambda f: f.name, field_info))
-        if enable_upsert:
-            self._valid_upsert(obj, field_names, bkey, exclusive_fields, update_fields)
-
-        exclusive_fields = set(exclusive_fields or {bkey}) & set(field_names)
-        update_fields = set(update_fields or (field_names - {bkey})) & set(field_names)
-        if enable_upsert and update_fields:
-            insert_ql = bulk_upsert_by_fields(
-                object_name, field_info, tgt_main_field,
-                exclusive_fields, update_fields
-            )
+        if commit_per_chunk:
+            for ql_chunk in qls:
+                await self.execute(ql_chunk)
         else:
-            insert_ql = bulk_insert_by_fields(object_name, field_info, tgt_main_field)
-
-        qls = []
-        self._collect_qls(structure.fit(data), insert_ql, chunksize, qls)
-        if self_link_dfs:
-            for update_field, (update_df, main_field) in self_link_dfs.items():
-                field = structure.fields[update_field]
-                update_ql = bulk_update_by_fields(
-                    object_name, [field], {update_field: main_field},
-                    [bkey], [update_field]
-                )
-                self._collect_qls(update_df, update_ql, chunksize, qls)
-
-        await self.execute(qls)
+            await self.execute(list(chain(*qls)))
 
     async def get_object(
         self,
@@ -1693,6 +1729,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         relation: Dict[str, pd.DataFrame] = None,
         chunksize: int = 500,
         match_fields: Iterable[str] = None,
+        commit_per_chunk: bool = False,
     ) -> None:
         """以事务执行基于DataFrame字段信息的批量更新数据
 
@@ -1709,57 +1746,23 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                     若有link property，则以property名为列名，提供在除source和target的列中
             chunksize: 单次最大行数
             match_fields: update的匹配列表，涉及的fields需出现在data或relation中，默认为业务主键
+            commit_per_chunk: 每次插入后是否提交事务，
+                            默认为False，即所有数据插入后再提交事务
+                            该参数仅在非start transaction上下文中生效
         """
         if data.empty:
             logger.info("data为空，无DML执行")
             return
 
-        if object_name in self.objects:
-            obj = self.objects[object_name]
-        else:
-            raise ObjectNotExist(
-                f'DeepModel对象[{object_name}]在当前应用不存在，无法更新数据'
-            )
-        if obj.external:
-            raise ExternalObjectReadOnly('外部对象只可读')
-
-        structure = ObjectStructure(name=obj.name, structure=obj.fields.values())
-        self._valid_data(data, object_name, relation, structure, check_required=False)
-        relation = relation or {}
-        bkey = await self._get_bkey(obj)
-        if bkey not in data.columns:
-            raise RequiredFieldUnfilled(f'缺少业务主键[{bkey}]')
-        # data拼接relation df
-        data = self._merge_relation(data, relation, structure, bkey)
-        # 从data中分离出self-link更新信息
-        data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
-        field_info, tgt_main_field = await self._collect_bulk_field_info(
-            object_name, structure, data, relation
-        )
-        field_names = set(map(lambda f: f.name, field_info))
-
-        if missing := (set(match_fields or [bkey]) - set(field_names)):
-            raise ValueError(f"match fields: {missing} 不在提供的数据中")
-
-        match_fields = set(match_fields or [bkey]) & set(field_names)
-        update_ql = bulk_update_by_fields(
-            object_name, field_info, tgt_main_field,
-            match_fields, field_names - match_fields
+        qls = await self._collect_bulk_qls(
+            object_name, data, relation, chunksize,
+            match_fields=match_fields, insert=False
         )
-        qls = []
-        self._collect_qls(structure.fit(data), update_ql, chunksize, qls)
-        if self_link_dfs:
-            for update_field, (update_df, main_field) in self_link_dfs.items():
-                field = structure.fields[update_field]
-                update_ql = bulk_update_by_fields(
-                    object_name, [field],
-                    {update_field: main_field},
-                    [bkey],
-                    [update_field]
-                )
-                self._collect_qls(update_df, update_ql, chunksize, qls)
-
-        await self.execute(qls)
+        if commit_per_chunk:
+            for ql_chunk in qls:
+                await self.execute(ql_chunk)
+        else:
+            await self.execute(list(chain(*qls)))
 
     @asynccontextmanager
     async def start_transaction(self, flatten: bool = False):
@@ -1799,7 +1802,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
 
         Important:
 
-            仅 :func:`insert_df` :func:`execute` 方法支持在事务中执行
+            仅 :func:`insert_df` :func:`update_df` :func:`execute` 方法支持在事务中执行
 
         """
         try:
@@ -1913,6 +1916,7 @@ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
             enable_upsert: bool = False,
             update_fields: Iterable[str] = None,
             exclusive_fields: Iterable[str] = None,
+            commit_per_chunk: bool = False,
         ) -> None:
             ...
 
@@ -1934,6 +1938,7 @@ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
             relation: Dict[str, pd.DataFrame] = None,
             chunksize: int = 500,
             match_fields: Iterable[str] = None,
+            commit_per_chunk: bool = False,
         ) -> None:
             ...
 

{deepfos-1.1.66 → deepfos-1.1.68}/deepfos/element/dimension.py

@@ -956,7 +956,7 @@ class AsyncDimension(ElementBase[DimensionAPI]):
 
         # -----------------------------------------------------------------------------
         # has shared member
-        df_shared: pd.DataFrame = df[df[shared_mbr_col]]
+        df_shared: pd.DataFrame = df[df[shared_mbr_col].astype(bool)]
         if self._strict:
             existed_mbrs = self._member_memo
 
@@ -989,7 +989,7 @@ class AsyncDimension(ElementBase[DimensionAPI]):
         # 在df中丢弃系统中已存在的共享节点
         df_shared_remain = df_shared[~df_shared.index.isin(dup_sharedmbrs)]
 
-        df_not_shared = df[~df[shared_mbr_col]]
+        df_not_shared = df[~df[shared_mbr_col].astype(bool)]
         existed_idx = df_not_shared[DFLT_NAME_COLUMN].isin(existed_mbrs)
         return df_not_shared.loc[existed_idx], \
             pd.concat([df_not_shared.loc[~existed_idx], df_shared_remain])