deepfos 1.1.75__tar.gz → 1.1.76__tar.gz

{deepfos-1.1.75 → deepfos-1.1.76}/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [1.1.76] - 2025-11-06
+
+### 更新
+
+* 维度增加dataTypeInfo字段
+* DeepModel增加error_on_empty_link
+* insert_df性能优化
+
+
 ## [1.1.75] - 2025-09-17
 
 ### 更新

{deepfos-1.1.75 → deepfos-1.1.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.75
+Version: 1.1.76
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team

{deepfos-1.1.75 → deepfos-1.1.76}/deepfos/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-09-18T10:14:56+0000",
+ "date": "2025-11-06T08:50:17+0000",
  "dirty": false,
  "error": null,
- "full-revisionid": "874420bc37e43a0b78366335290be2dbff468540",
- "version": "1.1.75"
+ "full-revisionid": "1f02db295b03468baac5e6df6dfd160ee0eba4a2",
+ "version": "1.1.76"
 }
 '''  # END VERSION_JSON
 

{deepfos-1.1.75 → deepfos-1.1.76}/deepfos/api/V1_2/models/dimension.py

@@ -877,6 +877,7 @@ class DimensionMemberNewDto(BaseModel):
 
 
 class DimensionMemberOperationDto(BaseModel):
+    dataTypeInfo: Optional[str] = None
     #: 变动类型
     moveType: Optional[str] = None
     #: 科目类型，科目类特有属性

{deepfos-1.1.75 → deepfos-1.1.76}/deepfos/api/models/dimension.py

@@ -542,6 +542,7 @@ class DimensionMemberListDto(BaseModel):
 
 
 class DimensionMemberOperationSw(BaseModel):
+    dataTypeInfo: Optional[str] = None
     #: 变动类型
     moveType: Optional[str] = None
     #: 支付类型,科目类型特有

{deepfos-1.1.75 → deepfos-1.1.76}/deepfos/element/deepmodel.py

@@ -304,20 +304,43 @@ class ObjectTypeFrame(BaseModel):
         return exclusive
 
 
-def _format_link(df: pd.DataFrame, link_name: str):
-    if all(pd.isnull(df['target'])):
-        return {'target': pd.NA}
+def _format_link(link_df_fit: pd.DataFrame, link_name: str):
+    if link_df_fit.empty:
+        return pd.Series(dtype=object), False
 
-    record = df.drop(columns=['source']).set_index('target')
-
-    if not record.index.is_unique:
+    if link_df_fit.duplicated(subset=['source', 'target']).any():
         raise MultiLinkTargetNotUnique(
             f'Multi Link: [{link_name}] relation dataframe中'
             f'source与target对应存在不唯一性'
         )
 
-    record = record.to_dict(orient='index')
-    return {'prop': record, 'target': list(record.keys())}
+    prop_cols = [col for col in link_df_fit.columns if col not in ['source', 'target']]
+    has_props = bool(prop_cols)
+
+    if has_props:
+        sources = link_df_fit['source'].values
+        targets = link_df_fit['target'].values
+        unique_sources, source_indices = np.unique(sources, return_inverse=True)
+        
+        prop_arrays = {col: link_df_fit[col].values for col in prop_cols}
+        result = {}
+
+        for i in range(len(unique_sources)):
+            idx = source_indices == i
+            source = unique_sources[i]
+            source_targets = targets[idx]
+
+            indices = np.where(idx)[0]
+            prop_dict = {
+                source_targets[j]: {col: prop_arrays[col][indices[j]] for col in prop_cols}
+                for j in range(len(source_targets))
+            }
+            result[source] = {'target': source_targets.tolist(), 'prop': prop_dict}
+        link = pd.Series(result, dtype=object)
+    else:
+        link = link_df_fit.groupby('source')['target'].agg(list)
+
+    return link, has_props
 
 
 class BaseField(PtrInfo):
@@ -486,7 +509,8 @@ def _iter_link_prop_assign(link, business_key, prop_name, prop_type, is_multi):
 def _iter_single_assign(
     field: PtrInfo,
     cast_type: str,
-    target_main_field: Dict[str, MainField]
+    target_main_field: Dict[str, MainField],
+    error_on_empty_link: bool = False
 ) -> str:
     """
     生成单字段赋值语句
@@ -495,6 +519,7 @@ def _iter_single_assign(
         field: 字段信息
         cast_type: 字段类型
         target_main_field: 目标字段信息
+        error_on_empty_link: 链接字段值不存在时是否抛出异常
 
     Returns:
         赋值语句
@@ -517,30 +542,38 @@ def _iter_single_assign(
     main_field = target_main_field[link]
 
     if main_field.props:
-        target = (
-            cast_type + "{" +
-            ",".join(
-                _iter_link_prop_assign(link, main_field.business_key, name,
-                                       field.prop_type[name], main_field.is_multi)
-                for name in main_field.props
-            ) + "}"
+        prop_assigns = ','.join(
+            _iter_link_prop_assign(link, main_field.business_key, name,
+                                   field.prop_type[name], main_field.is_multi)
+            for name in main_field.props
         )
+        prop_block = f" {{{prop_assigns}}}"
     else:
-        target = cast_type
+        prop_block = ""
 
     if main_field.is_multi:
+        link_value = f"each_{link}"
+    else:
+        link_value = f"(json_get(item, '{link}'))"
+
+    if error_on_empty_link:
+        link_expr = f"(<{cast_type}><std::str>{link_value}){prop_block}"
+    else:
+        link_expr = f"(select detached {cast_type}{prop_block}\nfilter .{main_field.business_key} = <std::str>{link_value})"
+
+    if main_field.is_multi:
+        if main_field.props:
+            target_source = f"json_get(item, '{link}', 'target')"
+        else:
+            target_source = f"item['{link}']"
+        
         assign += 'distinct (\n' + textwrap.indent(textwrap.dedent(f"""\
-        for each_{link} in json_array_unpack(json_get(item, '{link}', 'target'))
+        for each_{link} in json_array_unpack({target_source})
         union (
-            select detached {target} 
-            filter .{main_field.business_key} = <{main_field.type}>each_{link}
+            {link_expr}
         )"""), TAB) + '\n)'
     else:
-        assign += textwrap.dedent(f"""\
-        assert_single((
-            select detached {target}
-            filter .{main_field.business_key} = <{main_field.type}>(json_get(item, '{link}'))
-        ))""")
+        assign += link_expr
 
     return assign
 
@@ -549,9 +582,10 @@ def bulk_insert_by_fields(
     object_name: str,
     field_type: List[PtrInfo],
     target_main_field: Dict[str, MainField],
+    error_on_empty_link: bool = False,
 ):
     insert_assign_body = ','.join([
-        _iter_single_assign(field, field.type, target_main_field)
+        _iter_single_assign(field, field.type, target_main_field, error_on_empty_link)
         for field in field_type
     ])
     return textwrap.dedent(f"""
@@ -567,16 +601,17 @@ def bulk_upsert_by_fields(
     field_type: List[PtrInfo],
     target_main_field: Dict[str, MainField],
     exclusive_fields: Iterable[str],
-    update_fields: Iterable[str]
+    update_fields: Iterable[str],
+    error_on_empty_link: bool = False,
 ):
     conflict_on_fields = map(lambda n: f'.{n}', exclusive_fields)
     insert_assign_body = ','.join([
-        _iter_single_assign(field, field.type, target_main_field)
+        _iter_single_assign(field, field.type, target_main_field, error_on_empty_link)
         for field in field_type
     ])
     update_assign_body = ','.join(
         [
-            _iter_single_assign(field, field.type, target_main_field)
+            _iter_single_assign(field, field.type, target_main_field, error_on_empty_link)
             for field in field_type if field.name in update_fields
         ]
     )
@@ -599,9 +634,10 @@ def bulk_update_by_fields(
     target_main_field: Dict[str, MainField],
     match_fields: Iterable[str],
     update_fields: Iterable[str],
+    error_on_empty_link: bool = False,
 ):
     update_assign_body = ','.join([
-        _iter_single_assign(field, field.type, target_main_field)
+        _iter_single_assign(field, field.type, target_main_field, error_on_empty_link)
         for field in field_type if field.name in update_fields
     ])
 
@@ -1374,9 +1410,20 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
                 ]
             )
             link_df = temp_structure.fit(link_df)
-            link = link_df.groupby('source').apply(_format_link, link_name=name)
-            data = data.drop(columns=[name], errors='ignore')
-            data = data.join(link.to_frame(name), on=bkey)
+            link, has_props = _format_link(link_df, name)
+
+            if not has_props:
+                data = data.drop(columns=[name], errors='ignore')
+                data = data.join(link.to_frame(name), on=bkey, how='left')
+                mask = data[name].isna()
+                if mask.any():
+                    empty_series = pd.Series([[]] * mask.sum(), index=data[mask].index, dtype=object)
+                    data.loc[mask, name] = empty_series
+            else:
+                bkey_values = data[bkey].values
+                mapped_values = np.array([link.get(key, []) for key in bkey_values], dtype=object)
+                data[name] = mapped_values
+            
         return data
 
     async def _collect_bulk_qls(
@@ -1389,7 +1436,8 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         update_fields: Iterable[str] = None,
         exclusive_fields: Iterable[str] = None,
         match_fields: Iterable[str] = None,
-        insert: bool = True
+        insert: bool = True,
+        error_on_empty_link: bool = False
     ) -> List[List[QueryWithArgs]]:
         if object_name in self.objects:
             obj = self.objects[object_name]
@@ -1425,10 +1473,10 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             if enable_upsert and update_fields:
                 bulk_ql = bulk_upsert_by_fields(
                     object_name, field_info, tgt_main_field,
-                    exclusive_fields, update_fields
+                    exclusive_fields, update_fields, error_on_empty_link
                 )
             else:
-                bulk_ql = bulk_insert_by_fields(object_name, field_info, tgt_main_field)
+                bulk_ql = bulk_insert_by_fields(object_name, field_info, tgt_main_field, error_on_empty_link)
         else:
             if missing := (set(match_fields or [bkey]) - set(field_names)):
                 raise ValueError(f"match fields: {missing} 不在提供的数据中")
@@ -1437,7 +1485,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             if to_upd := (field_names - match_fields):
                 bulk_ql = bulk_update_by_fields(
                     object_name, field_info, tgt_main_field,
-                    match_fields, to_upd
+                    match_fields, to_upd, error_on_empty_link
                 )
             else:
                 bulk_ql = None
@@ -1473,6 +1521,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         update_fields: Iterable[str] = None,
         exclusive_fields: Iterable[str] = None,
         commit_per_chunk: bool = False,
+        error_on_empty_link: bool = False,
     ) -> None:
         """以事务执行基于DataFrame字段信息的批量插入数据
 
@@ -1496,6 +1545,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
             commit_per_chunk: 每次插入后是否提交事务，
                             默认为False，即所有数据插入后再提交事务
                             该参数仅在非start transaction上下文中生效
+            error_on_empty_link: 链接字段值不存在时是否抛出异常，
+                            默认为False，即不检查链接目标是否存在
+                            当设置为True时，会检查链接目标是否存在，不存在则抛出异常
 
         Notes:
 
@@ -1571,7 +1623,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
         qls = await self._collect_bulk_qls(
             object_name, data, relation, chunksize,
             enable_upsert, update_fields, exclusive_fields,
-            insert=True
+            insert=True, error_on_empty_link=error_on_empty_link
         )
         if commit_per_chunk:
             for ql_chunk in qls:
@@ -1918,6 +1970,7 @@ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
             update_fields: Iterable[str] = None,
             exclusive_fields: Iterable[str] = None,
             commit_per_chunk: bool = False,
+            error_on_empty_link: bool = False,
         ) -> None:
             ...
 

{deepfos-1.1.75 → deepfos-1.1.76}/deepfos/element/dimension.py

@@ -1162,12 +1162,25 @@ def _validate_df_for_dimension(df: pd.DataFrame):
             f"You have null value in dataframe. "
             f"column: [{DFLT_NAME_COLUMN}], index: {null_index}.")
 
-    if DFLT_PNAME_COLUMN not in df.columns:
-        raise ValueError(f"Missing column [{DFLT_PNAME_COLUMN}] in dataframe.")
+    col_parent_name = None
+    col_shared_member = None
+
+    if DFLT_PNAME_COLUMN in df.columns:
+        col_parent_name = DFLT_PNAME_COLUMN
+    elif DFLT_PNAME_COLUMN_V12 in df.columns:
+        col_parent_name = DFLT_PNAME_COLUMN_V12
 
     if SHAREDMEMBER in df.columns:
+        col_shared_member = SHAREDMEMBER
+    elif SHAREDMEMBERV12 in df.columns:
+        col_shared_member = SHAREDMEMBERV12
+
+    if col_parent_name is None:
+        raise ValueError(f"Missing column [{DFLT_PNAME_COLUMN}] or [{DFLT_PNAME_COLUMN_V12}] in dataframe.")
+
+    if col_shared_member is not None:
         unique_df = df.groupby(
-            [DFLT_NAME_COLUMN, DFLT_PNAME_COLUMN, SHAREDMEMBER],
+            [DFLT_NAME_COLUMN, col_parent_name, col_shared_member],
             as_index=False
         ).size()
         duplicated = unique_df[unique_df['size'] > 1]

{deepfos-1.1.75 → deepfos-1.1.76}/deepfos.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deepfos
-Version: 1.1.75
+Version: 1.1.76
 Summary: Collecions of useful and handy tools for deepfos platform
 Home-page: http://py.deepfos.com
 Author: deepfos-python-team