deepfos 1.1.54__tar.gz → 1.1.56__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deepfos-1.1.54 → deepfos-1.1.56}/CHANGELOG.md +19 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/PKG-INFO +1 -1
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/_version.py +3 -3
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/deepmodel.py +294 -149
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/finmodel.py +220 -61
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/serutils.py +34 -9
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/sysutils.py +31 -11
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos.egg-info/PKG-INFO +1 -1
- {deepfos-1.1.54 → deepfos-1.1.56}/.gitattributes +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/.gitee/ISSUE_GUIDELINES.md +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/.gitee/ISSUE_TEMPLATE.md +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/.gitignore +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/MANIFEST.in +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/README.md +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/algo/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/algo/graph.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_1/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_1/business_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_1/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_1/models/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_1/models/business_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_1/models/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_2/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_2/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_2/models/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/V1_2/models/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/account.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/accounting_engines.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/app.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/approval_process.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/base.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/business_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/consolidation.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/consolidation_process.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/datatable.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/deep_pipeline.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/deepconnector.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/deepfos_task.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/deepmodel.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/financial_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/journal_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/journal_template.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/memory_financial_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/account.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/accounting_engines.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/app.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/approval_process.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/base.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/business_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/consolidation.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/consolidation_process.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/datatable_mysql.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/deep_pipeline.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/deepconnector.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/deepfos_task.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/deepmodel.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/financial_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/journal_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/journal_template.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/memory_financial_model.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/platform.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/python.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/reconciliation_engine.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/reconciliation_report.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/role_strategy.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/smartlist.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/space.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/system.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/variable.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/models/workflow.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/platform.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/python.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/reconciliation_engine.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/reconciliation_report.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/role_strategy.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/smartlist.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/space.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/system.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/variable.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/api/workflow.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/boost/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/boost/jstream.c +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/boost/jstream.pyx +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/boost/pandas.c +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/boost/pandas.pyx +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/boost/py_jstream.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/boost/py_pandas.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/cache.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/config.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/_base.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/constants.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/cube.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/formula.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/syscube.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/typing.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/cube/utils.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/_base.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/dimcreator.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/dimexpr.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/dimmember.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/eledimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/filters.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/dimension/sysdimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/logictable/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/logictable/_cache.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/logictable/_operator.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/logictable/nodemixin.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/logictable/sqlcondition.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/core/logictable/tablemodel.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/cipher.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/clickhouse.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/connector.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/daclickhouse.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/dameng.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/damysql.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/dbkits.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/deepengine.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/deepmodel.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/deepmodel_kingbase.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/edb.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/gauss.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/kingbase.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/mysql.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/oracle.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/postgresql.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/sqlserver.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/db/utils.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/accounting.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/apvlprocess.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/base.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/bizmodel.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/datatable.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/deep_pipeline.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/deepconnector.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/dimension.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/fact_table.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/journal.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/journal_template.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/pyscript.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/reconciliation.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/rolestrategy.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/smartlist.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/variable.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/element/workflow.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/exceptions/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/exceptions/hook.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lazy.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/__init__.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/_javaobj.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/asynchronous.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/concurrency.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/constant.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/decorator.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/deepchart.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/deepux.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/discovery.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/edb_lexer.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/eureka.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/filterparser.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/httpcli.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/jsonstreamer.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/msg.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/nacos.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/patch.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/redis.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/stopwatch.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/subtask.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/lib/utils.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/local.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/options.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos/translation.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos.egg-info/SOURCES.txt +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos.egg-info/dependency_links.txt +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos.egg-info/not-zip-safe +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos.egg-info/requires.txt +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/deepfos.egg-info/top_level.txt +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/requirements.txt +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/setup.cfg +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/setup.py +0 -0
- {deepfos-1.1.54 → deepfos-1.1.56}/versioneer.py +0 -0
|
@@ -1,3 +1,22 @@
|
|
|
1
|
+
## [1.1.56] - 2025-04-27
|
|
2
|
+
|
|
3
|
+
### 新增
|
|
4
|
+
|
|
5
|
+
* 财务模型支持complement_save_unpivot
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
## [1.1.55] - 2025-04-25
|
|
9
|
+
|
|
10
|
+
### 新增
|
|
11
|
+
* DeepModel支持update_df
|
|
12
|
+
* 财务模型支持complement_save
|
|
13
|
+
|
|
14
|
+
### 变更
|
|
15
|
+
|
|
16
|
+
* DeepModel事务内设置globals不生效问题修正
|
|
17
|
+
* DeepModel兼容线上线下模式的json类型数据的处理,保证insert_df和query_df的读写格式一致性
|
|
18
|
+
|
|
19
|
+
|
|
1
20
|
## [1.1.54] - 2025-03-27
|
|
2
21
|
|
|
3
22
|
### 变更
|
|
@@ -8,11 +8,11 @@ import json
|
|
|
8
8
|
|
|
9
9
|
version_json = '''
|
|
10
10
|
{
|
|
11
|
-
"date": "2025-
|
|
11
|
+
"date": "2025-04-28T08:53:07+0000",
|
|
12
12
|
"dirty": false,
|
|
13
13
|
"error": null,
|
|
14
|
-
"full-revisionid": "
|
|
15
|
-
"version": "1.1.
|
|
14
|
+
"full-revisionid": "65654605edcd2831ed325b40b62f71c933313017",
|
|
15
|
+
"version": "1.1.56"
|
|
16
16
|
}
|
|
17
17
|
''' # END VERSION_JSON
|
|
18
18
|
|
|
@@ -128,6 +128,7 @@ class ObjectElement(ObjectParam):
|
|
|
128
128
|
class QueryWithArgs(BaseModel):
|
|
129
129
|
commands: str
|
|
130
130
|
kwargs: Dict[str, Any] = Field(default_factory=dict)
|
|
131
|
+
globals: Dict[str, Any] = Field(default_factory=dict)
|
|
131
132
|
|
|
132
133
|
|
|
133
134
|
class MainField(NamedTuple):
|
|
@@ -370,11 +371,7 @@ class FieldJson(BaseField):
|
|
|
370
371
|
df[field_name] = df[field_name].apply(self.format_json)
|
|
371
372
|
|
|
372
373
|
def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
|
|
373
|
-
|
|
374
|
-
# since json value will be converted to json string(type: str)
|
|
375
|
-
# in edgedb python protocol
|
|
376
|
-
if not direct_access:
|
|
377
|
-
df[field_name] = df[field_name].apply(self.format_json)
|
|
374
|
+
pass
|
|
378
375
|
|
|
379
376
|
|
|
380
377
|
class FieldInt(BaseField):
|
|
@@ -488,6 +485,9 @@ def _iter_single_assign(
|
|
|
488
485
|
assign_string = f"{field.name} := "
|
|
489
486
|
# 设置标量值
|
|
490
487
|
if field.name not in target_main_field:
|
|
488
|
+
if field.is_multi:
|
|
489
|
+
return assign_string + f"json_array_unpack(item['{field.name}'])"
|
|
490
|
+
|
|
491
491
|
assign_string += f"<{cast_type}>"
|
|
492
492
|
|
|
493
493
|
if cast_type in NEED_CAST_STR:
|
|
@@ -590,10 +590,10 @@ def bulk_upsert_by_fields(
|
|
|
590
590
|
|
|
591
591
|
def bulk_update_by_fields(
|
|
592
592
|
object_name: str,
|
|
593
|
-
business_key: str,
|
|
594
593
|
field_type: List[PtrInfo],
|
|
595
594
|
target_main_field: Dict[str, MainField],
|
|
596
|
-
|
|
595
|
+
match_fields: Iterable[str],
|
|
596
|
+
update_fields: Iterable[str],
|
|
597
597
|
):
|
|
598
598
|
update_assign_body = ','.join(
|
|
599
599
|
[
|
|
@@ -602,11 +602,20 @@ def bulk_update_by_fields(
|
|
|
602
602
|
]
|
|
603
603
|
)
|
|
604
604
|
|
|
605
|
+
field_type_map = {field.name: field.type for field in field_type}
|
|
606
|
+
|
|
607
|
+
match_str = " and ".join(
|
|
608
|
+
[
|
|
609
|
+
f".{name} = <{field_type_map.get(name, 'std::str')}>item['{name}']"
|
|
610
|
+
for name in match_fields
|
|
611
|
+
]
|
|
612
|
+
)
|
|
613
|
+
|
|
605
614
|
return f"""
|
|
606
615
|
with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
|
|
607
616
|
for item in json_array_unpack(raw_data) union (
|
|
608
617
|
update {object_name}
|
|
609
|
-
filter
|
|
618
|
+
filter {match_str}
|
|
610
619
|
set {{
|
|
611
620
|
{update_assign_body}
|
|
612
621
|
}}
|
|
@@ -765,8 +774,12 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
765
774
|
"""DeepModel元素
|
|
766
775
|
|
|
767
776
|
Args:
|
|
768
|
-
direct_access:
|
|
777
|
+
direct_access: 是否使用直连模式,默认为True
|
|
778
|
+
会结合OPTION.edgedb.dsn是否有值决定是否使用直连模式,如无值,则仍为非直连模式
|
|
779
|
+
直连模式下,会使用edgedb-python库直连edgedb server,
|
|
780
|
+
否则会使用DeepModel组件API进行操作
|
|
769
781
|
pg_dsn: PG连接信息
|
|
782
|
+
|
|
770
783
|
"""
|
|
771
784
|
__mangle_docs__ = False
|
|
772
785
|
|
|
@@ -774,9 +787,12 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
774
787
|
self._txn_ = ContextVar('QLTXN')
|
|
775
788
|
self.appmodule = f"app{OPTION.api.header['app']}"
|
|
776
789
|
self.spacemodule = f"space{OPTION.api.header['space']}"
|
|
777
|
-
self.direct_access = direct_access
|
|
790
|
+
self.direct_access = direct_access and bool(OPTION.edgedb.dsn)
|
|
791
|
+
if not self.direct_access:
|
|
792
|
+
logger.debug('当前DeepModel为非直连模式')
|
|
778
793
|
self.alias = AliasGenerator()
|
|
779
794
|
self.pg_dsn = pg_dsn
|
|
795
|
+
self._globals = {}
|
|
780
796
|
|
|
781
797
|
@future_property
|
|
782
798
|
async def client(self):
|
|
@@ -799,12 +815,12 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
799
815
|
dbname=dbname
|
|
800
816
|
)
|
|
801
817
|
if user_id := OPTION.api.header.get('user'):
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
818
|
+
default_globals = {
|
|
819
|
+
f'{self.spacemodule}::current_user_id':
|
|
820
|
+
user_id
|
|
821
|
+
}
|
|
822
|
+
client = client.with_globals(**default_globals)
|
|
823
|
+
self._globals = client._options.state._globals
|
|
808
824
|
return client
|
|
809
825
|
|
|
810
826
|
@future_property
|
|
@@ -899,12 +915,18 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
899
915
|
"""
|
|
900
916
|
|
|
901
917
|
if self.direct_access:
|
|
902
|
-
logger.opt(lazy=True).debug(f"Query: [{ql}]
|
|
918
|
+
logger.opt(lazy=True).debug(f"Query: [{ql}],\n"
|
|
919
|
+
f"kwargs: [{kwargs}],\n"
|
|
920
|
+
f"globals: [{self._globals}].")
|
|
903
921
|
_, result = await self.client.query(ql, **kwargs)
|
|
904
922
|
return result
|
|
905
923
|
|
|
906
|
-
result = await
|
|
907
|
-
|
|
924
|
+
result = await self._http_query(ql, **kwargs)
|
|
925
|
+
field_info = {
|
|
926
|
+
fi.name: fi.type if fi.fields is None else fi.fields
|
|
927
|
+
for fi in result.objectInfos[0].fields
|
|
928
|
+
} if result.objectInfos else {}
|
|
929
|
+
return serutils.deserialize(result.json_, field_info)
|
|
908
930
|
|
|
909
931
|
async def query(self, ql: str, **kwargs) -> List[Any]:
|
|
910
932
|
"""执行ql查询语句,得到序列化后的结果
|
|
@@ -934,7 +956,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
934
956
|
|
|
935
957
|
"""
|
|
936
958
|
if self.direct_access:
|
|
937
|
-
logger.opt(lazy=True).debug(f"Query: [{ql}]
|
|
959
|
+
logger.opt(lazy=True).debug(f"Query: [{ql}],\n"
|
|
960
|
+
f"kwargs: [{kwargs}],\n"
|
|
961
|
+
f"globals: [{self._globals}].")
|
|
938
962
|
frame_desc, result = await self.client.query(ql, **kwargs)
|
|
939
963
|
return serutils.serialize(
|
|
940
964
|
result, ctx=serutils.Context(frame_desc=frame_desc)
|
|
@@ -995,7 +1019,7 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
995
1019
|
else:
|
|
996
1020
|
result = await self._http_query(ql, **kwargs)
|
|
997
1021
|
# No object structure info
|
|
998
|
-
if result.objectInfos
|
|
1022
|
+
if not result.objectInfos:
|
|
999
1023
|
return pd.DataFrame(data=result.json_)
|
|
1000
1024
|
|
|
1001
1025
|
data = pd.DataFrame(
|
|
@@ -1015,6 +1039,10 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1015
1039
|
query_object.__doc__ = query_object.__doc__ + DOC_ARGS_KWARGS
|
|
1016
1040
|
query_df.__doc__ = query_df.__doc__ + DOC_ARGS_KWARGS
|
|
1017
1041
|
|
|
1042
|
+
def _ensure_client(self):
|
|
1043
|
+
if self.direct_access:
|
|
1044
|
+
self.client # noqa
|
|
1045
|
+
|
|
1018
1046
|
@txn_support
|
|
1019
1047
|
async def execute(
|
|
1020
1048
|
self,
|
|
@@ -1030,8 +1058,11 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1030
1058
|
会自动用作所有string形式ql的参数
|
|
1031
1059
|
|
|
1032
1060
|
"""
|
|
1061
|
+
self._ensure_client()
|
|
1033
1062
|
if isinstance(qls, str):
|
|
1034
|
-
qls_with_args = [QueryWithArgs(
|
|
1063
|
+
qls_with_args = [QueryWithArgs(
|
|
1064
|
+
commands=qls, kwargs=kwargs, globals=self._globals
|
|
1065
|
+
)]
|
|
1035
1066
|
else:
|
|
1036
1067
|
qls_with_args = []
|
|
1037
1068
|
seen_kwargs_key = set()
|
|
@@ -1049,7 +1080,9 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1049
1080
|
seen_kwargs_key = seen_kwargs_key.union(ql.kwargs.keys())
|
|
1050
1081
|
|
|
1051
1082
|
elif isinstance(ql, str):
|
|
1052
|
-
qls_with_args.append(QueryWithArgs(
|
|
1083
|
+
qls_with_args.append(QueryWithArgs(
|
|
1084
|
+
commands=ql, kwargs=kwargs, globals=self._globals
|
|
1085
|
+
))
|
|
1053
1086
|
else:
|
|
1054
1087
|
raise TypeError(f'qls参数中出现类型非法成员:{type(ql)}')
|
|
1055
1088
|
|
|
@@ -1089,12 +1122,25 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1089
1122
|
async with tx:
|
|
1090
1123
|
for ql in qls_with_args:
|
|
1091
1124
|
logger.opt(lazy=True).debug(
|
|
1092
|
-
f"Execute QL: [{ql.commands}],
|
|
1125
|
+
f"Execute QL: [{ql.commands}],"
|
|
1126
|
+
f"\nkwargs: [{ql.kwargs}],"
|
|
1127
|
+
f"\nglobals: [{ql.globals}]."
|
|
1093
1128
|
)
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1129
|
+
if ql.globals:
|
|
1130
|
+
bak_cli = tx._client
|
|
1131
|
+
tx._client = tx._client.with_globals(**ql.globals)
|
|
1132
|
+
try:
|
|
1133
|
+
desc, affected = await tx.execute(ql.commands, **ql.kwargs)
|
|
1134
|
+
result.append(serutils.serialize(
|
|
1135
|
+
affected, ctx=serutils.Context(frame_desc=desc)
|
|
1136
|
+
))
|
|
1137
|
+
finally:
|
|
1138
|
+
tx._client = bak_cli
|
|
1139
|
+
else:
|
|
1140
|
+
desc, affected = await tx.execute(ql.commands, **ql.kwargs)
|
|
1141
|
+
result.append(serutils.serialize(
|
|
1142
|
+
affected, ctx=serutils.Context(frame_desc=desc)
|
|
1143
|
+
))
|
|
1098
1144
|
if len(result) == 1:
|
|
1099
1145
|
return result[0]
|
|
1100
1146
|
return result
|
|
@@ -1176,6 +1222,128 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1176
1222
|
if exclusive not in valid_exclusive:
|
|
1177
1223
|
raise ValueError(f"exclusive fields: {exclusive_fields} 没有相应唯一约束")
|
|
1178
1224
|
|
|
1225
|
+
async def _get_bkey(
|
|
1226
|
+
self,
|
|
1227
|
+
obj: Union[ObjectTypeFrame, TargetField],
|
|
1228
|
+
source: str = None,
|
|
1229
|
+
name: str = None
|
|
1230
|
+
) -> str:
|
|
1231
|
+
# 如可在object结构的annotations中取业务主键,则优先取,否则走接口
|
|
1232
|
+
if obj.info and BUSINESS_KEY in obj.info:
|
|
1233
|
+
return obj.info[BUSINESS_KEY]
|
|
1234
|
+
elif (code := obj.normalized_name) in self.model_objects:
|
|
1235
|
+
return self.model_objects[code].businessKey
|
|
1236
|
+
|
|
1237
|
+
assert isinstance(obj, TargetField)
|
|
1238
|
+
# Link 至非本应用对象,需单独查询
|
|
1239
|
+
tgt = ObjectElement.construct_from(self.model_objects[source]).links[name]
|
|
1240
|
+
tgt_model_info = await self.async_api.object.info(
|
|
1241
|
+
app=tgt.targetApp, object_code=tgt.targetObjectCode
|
|
1242
|
+
)
|
|
1243
|
+
return tgt_model_info.businessKey
|
|
1244
|
+
|
|
1245
|
+
async def _collect_bulk_field_info(self, object_name, structure, data, relation):
|
|
1246
|
+
field_info = []
|
|
1247
|
+
tgt_main_field = {}
|
|
1248
|
+
for field in structure.fields.values():
|
|
1249
|
+
if field.name not in data.columns:
|
|
1250
|
+
continue
|
|
1251
|
+
|
|
1252
|
+
field_info.append(field)
|
|
1253
|
+
|
|
1254
|
+
if not field.is_link:
|
|
1255
|
+
continue
|
|
1256
|
+
|
|
1257
|
+
is_multi = field.is_multi_link
|
|
1258
|
+
name = field.name
|
|
1259
|
+
# 链接至其他对象,记录目标对象信息
|
|
1260
|
+
if is_multi:
|
|
1261
|
+
link_props = set(relation[name].columns).intersection(field.props)
|
|
1262
|
+
else:
|
|
1263
|
+
link_props = set(
|
|
1264
|
+
c[len(f'{name}@')::]
|
|
1265
|
+
for c in data.columns if c.startswith(f'{name}@')
|
|
1266
|
+
).intersection(field.props)
|
|
1267
|
+
tgt_bkey = await self._get_bkey(field.target, object_name, name)
|
|
1268
|
+
tgt_main_field[name] = MainField(tgt_bkey, is_multi, link_props)
|
|
1269
|
+
return field_info, tgt_main_field
|
|
1270
|
+
|
|
1271
|
+
def _collect_qls(
|
|
1272
|
+
self,
|
|
1273
|
+
data: pd.DataFrame,
|
|
1274
|
+
ql: str,
|
|
1275
|
+
chunksize: int,
|
|
1276
|
+
qls: List[QueryWithArgs]
|
|
1277
|
+
):
|
|
1278
|
+
self._ensure_client()
|
|
1279
|
+
for i in range(0, len(data), chunksize):
|
|
1280
|
+
part = data.iloc[i: i + chunksize]
|
|
1281
|
+
kw_name = self.alias.get(BATCH_INSERT_KW)
|
|
1282
|
+
qls.append(QueryWithArgs(
|
|
1283
|
+
commands=ql.replace(
|
|
1284
|
+
f'${BATCH_INSERT_KW}', f'${kw_name}'
|
|
1285
|
+
),
|
|
1286
|
+
kwargs={kw_name: part.to_json(
|
|
1287
|
+
orient='records', double_precision=15,
|
|
1288
|
+
force_ascii=False, default_handler=str
|
|
1289
|
+
)},
|
|
1290
|
+
globals=self._globals
|
|
1291
|
+
))
|
|
1292
|
+
|
|
1293
|
+
@staticmethod
|
|
1294
|
+
def _split_self_link(data, relation, structure, bkey):
|
|
1295
|
+
self_link_dfs = {}
|
|
1296
|
+
for name in structure.self_link_fields:
|
|
1297
|
+
field = structure.fields[name]
|
|
1298
|
+
if (link_df := relation.get(name)) is not None:
|
|
1299
|
+
link_props = set(link_df.columns).intersection(field.props)
|
|
1300
|
+
self_link_dfs[name] = (
|
|
1301
|
+
structure.fit(data[[bkey, name]]),
|
|
1302
|
+
MainField(bkey, field.is_multi_link, link_props)
|
|
1303
|
+
)
|
|
1304
|
+
data = data.drop(columns=[name])
|
|
1305
|
+
elif name in data.columns:
|
|
1306
|
+
link_prop_cols = []
|
|
1307
|
+
link_props = []
|
|
1308
|
+
|
|
1309
|
+
for col in data.columns:
|
|
1310
|
+
if (
|
|
1311
|
+
col.startswith(f'{name}@')
|
|
1312
|
+
and ((prop_name := col[len(f'{name}@')::]) in field.props)
|
|
1313
|
+
):
|
|
1314
|
+
link_prop_cols.append(col)
|
|
1315
|
+
link_props.append(prop_name)
|
|
1316
|
+
|
|
1317
|
+
self_link_dfs[name] = (
|
|
1318
|
+
structure.fit(data[[bkey, name, *link_prop_cols]]),
|
|
1319
|
+
MainField(bkey, field.is_multi_link, link_props)
|
|
1320
|
+
)
|
|
1321
|
+
data = data.drop(columns=[name, *link_prop_cols])
|
|
1322
|
+
return data, self_link_dfs
|
|
1323
|
+
|
|
1324
|
+
@staticmethod
|
|
1325
|
+
def _merge_relation(data, relation, structure, bkey):
|
|
1326
|
+
for name, link_df in relation.items():
|
|
1327
|
+
if name not in structure.fields:
|
|
1328
|
+
continue
|
|
1329
|
+
field = structure.fields[name]
|
|
1330
|
+
valid_cols = list({'source', 'target', *field.props} & set(link_df.columns))
|
|
1331
|
+
link_df = link_df[valid_cols]
|
|
1332
|
+
# for fit only
|
|
1333
|
+
temp_structure = ObjectStructure(
|
|
1334
|
+
field.type,
|
|
1335
|
+
[
|
|
1336
|
+
PtrInfo(name='source', target=TargetField(name='std::str')),
|
|
1337
|
+
PtrInfo(name='target', target=TargetField(name='std::str')),
|
|
1338
|
+
*[PtrInfo(**prop.dict()) for prop in field.properties]
|
|
1339
|
+
]
|
|
1340
|
+
)
|
|
1341
|
+
link_df = temp_structure.fit(link_df)
|
|
1342
|
+
link = link_df.groupby('source').apply(_format_link, link_name=name)
|
|
1343
|
+
data = data.drop(columns=[name], errors='ignore')
|
|
1344
|
+
data = data.join(link.to_frame(name), on=bkey)
|
|
1345
|
+
return data
|
|
1346
|
+
|
|
1179
1347
|
@txn_support
|
|
1180
1348
|
async def insert_df(
|
|
1181
1349
|
self,
|
|
@@ -1297,38 +1465,15 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1297
1465
|
data = self._merge_relation(data, relation, structure, bkey)
|
|
1298
1466
|
# 从data中分离出self-link更新信息
|
|
1299
1467
|
data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
# 准备bulk insert所需field信息
|
|
1304
|
-
for field in structure.fields.values():
|
|
1305
|
-
if field.name not in data.columns:
|
|
1306
|
-
continue
|
|
1307
|
-
|
|
1308
|
-
field_info.append(field)
|
|
1309
|
-
|
|
1310
|
-
if not field.is_link:
|
|
1311
|
-
continue
|
|
1312
|
-
|
|
1313
|
-
is_multi = field.is_multi_link
|
|
1314
|
-
name = field.name
|
|
1315
|
-
# 链接至其他对象,记录目标对象信息
|
|
1316
|
-
if is_multi:
|
|
1317
|
-
link_props = set(relation[name].columns).intersection(field.props)
|
|
1318
|
-
else:
|
|
1319
|
-
link_props = set(
|
|
1320
|
-
c[len(f'{name}@')::]
|
|
1321
|
-
for c in data.columns if c.startswith(f'{name}@')
|
|
1322
|
-
).intersection(field.props)
|
|
1323
|
-
tgt_bkey = await self._get_bkey(field.target, object_name, name)
|
|
1324
|
-
tgt_main_field[name] = MainField(tgt_bkey, is_multi, link_props)
|
|
1325
|
-
|
|
1468
|
+
field_info, tgt_main_field = await self._collect_bulk_field_info(
|
|
1469
|
+
object_name, structure, data, relation
|
|
1470
|
+
)
|
|
1326
1471
|
field_names = set(map(lambda f: f.name, field_info))
|
|
1327
1472
|
if enable_upsert:
|
|
1328
1473
|
self._valid_upsert(obj, field_names, bkey, exclusive_fields, update_fields)
|
|
1329
1474
|
|
|
1330
1475
|
exclusive_fields = set(exclusive_fields or {bkey}) & set(field_names)
|
|
1331
|
-
update_fields = set(update_fields or (field_names- {bkey})) & set(field_names)
|
|
1476
|
+
update_fields = set(update_fields or (field_names - {bkey})) & set(field_names)
|
|
1332
1477
|
if enable_upsert and update_fields:
|
|
1333
1478
|
insert_ql = bulk_upsert_by_fields(
|
|
1334
1479
|
object_name, field_info, tgt_main_field,
|
|
@@ -1343,30 +1488,13 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1343
1488
|
for update_field, (update_df, main_field) in self_link_dfs.items():
|
|
1344
1489
|
field = structure.fields[update_field]
|
|
1345
1490
|
update_ql = bulk_update_by_fields(
|
|
1346
|
-
object_name,
|
|
1347
|
-
|
|
1491
|
+
object_name, [field], {update_field: main_field},
|
|
1492
|
+
[bkey], [update_field]
|
|
1348
1493
|
)
|
|
1349
1494
|
self._collect_qls(update_df, update_ql, chunksize, qls)
|
|
1350
1495
|
|
|
1351
1496
|
await self.execute(qls)
|
|
1352
1497
|
|
|
1353
|
-
def _collect_qls(
|
|
1354
|
-
self,
|
|
1355
|
-
data: pd.DataFrame,
|
|
1356
|
-
ql: str,
|
|
1357
|
-
chunksize: int,
|
|
1358
|
-
qls: List[QueryWithArgs]
|
|
1359
|
-
):
|
|
1360
|
-
for i in range(0, len(data), chunksize):
|
|
1361
|
-
part = data.iloc[i: i + chunksize]
|
|
1362
|
-
kw_name = self.alias.get(BATCH_INSERT_KW)
|
|
1363
|
-
qls.append(QueryWithArgs(
|
|
1364
|
-
commands=ql.replace(
|
|
1365
|
-
f'${BATCH_INSERT_KW}', f'${kw_name}'
|
|
1366
|
-
),
|
|
1367
|
-
kwargs={kw_name: part.to_json(orient='records', double_precision=15)}
|
|
1368
|
-
))
|
|
1369
|
-
|
|
1370
1498
|
async def get_object(
|
|
1371
1499
|
self,
|
|
1372
1500
|
object_name: str,
|
|
@@ -1395,80 +1523,6 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1395
1523
|
|
|
1396
1524
|
return format_obj(objs[0])
|
|
1397
1525
|
|
|
1398
|
-
async def _get_bkey(
|
|
1399
|
-
self,
|
|
1400
|
-
obj: Union[ObjectTypeFrame, TargetField],
|
|
1401
|
-
source: str = None,
|
|
1402
|
-
name: str = None
|
|
1403
|
-
) -> str:
|
|
1404
|
-
# 如可在object结构的annotations中取业务主键,则优先取,否则走接口
|
|
1405
|
-
if obj.info and BUSINESS_KEY in obj.info:
|
|
1406
|
-
return obj.info[BUSINESS_KEY]
|
|
1407
|
-
elif (code := obj.normalized_name) in self.model_objects:
|
|
1408
|
-
return self.model_objects[code].businessKey
|
|
1409
|
-
|
|
1410
|
-
assert isinstance(obj, TargetField)
|
|
1411
|
-
# Link 至非本应用对象,需单独查询
|
|
1412
|
-
tgt = ObjectElement.construct_from(self.model_objects[source]).links[name]
|
|
1413
|
-
tgt_model_info = await self.async_api.object.info(
|
|
1414
|
-
app=tgt.targetApp, object_code=tgt.targetObjectCode
|
|
1415
|
-
)
|
|
1416
|
-
return tgt_model_info.businessKey
|
|
1417
|
-
|
|
1418
|
-
@staticmethod
|
|
1419
|
-
def _split_self_link(data, relation, structure, bkey):
|
|
1420
|
-
self_link_dfs = {}
|
|
1421
|
-
for name in structure.self_link_fields:
|
|
1422
|
-
field = structure.fields[name]
|
|
1423
|
-
if (link_df := relation.get(name)) is not None:
|
|
1424
|
-
link_props = set(link_df.columns).intersection(field.props)
|
|
1425
|
-
self_link_dfs[name] = (
|
|
1426
|
-
structure.fit(data[[bkey, name]]),
|
|
1427
|
-
MainField(bkey, field.is_multi_link, link_props)
|
|
1428
|
-
)
|
|
1429
|
-
data = data.drop(columns=[name])
|
|
1430
|
-
elif name in data.columns:
|
|
1431
|
-
link_prop_cols = []
|
|
1432
|
-
link_props = []
|
|
1433
|
-
|
|
1434
|
-
for col in data.columns:
|
|
1435
|
-
if (
|
|
1436
|
-
col.startswith(f'{name}@')
|
|
1437
|
-
and ((prop_name := col[len(f'{name}@')::]) in field.props)
|
|
1438
|
-
):
|
|
1439
|
-
link_prop_cols.append(col)
|
|
1440
|
-
link_props.append(prop_name)
|
|
1441
|
-
|
|
1442
|
-
self_link_dfs[name] = (
|
|
1443
|
-
structure.fit(data[[bkey, name, *link_prop_cols]]),
|
|
1444
|
-
MainField(bkey, field.is_multi_link, link_props)
|
|
1445
|
-
)
|
|
1446
|
-
data = data.drop(columns=[name, *link_prop_cols])
|
|
1447
|
-
return data, self_link_dfs
|
|
1448
|
-
|
|
1449
|
-
@staticmethod
|
|
1450
|
-
def _merge_relation(data, relation, structure, bkey):
|
|
1451
|
-
for name, link_df in relation.items():
|
|
1452
|
-
if name not in structure.fields:
|
|
1453
|
-
continue
|
|
1454
|
-
field = structure.fields[name]
|
|
1455
|
-
valid_cols = list({'source', 'target', *field.props} & set(link_df.columns))
|
|
1456
|
-
link_df = link_df[valid_cols]
|
|
1457
|
-
# for fit only
|
|
1458
|
-
temp_structure = ObjectStructure(
|
|
1459
|
-
field.type,
|
|
1460
|
-
[
|
|
1461
|
-
PtrInfo(name='source', target=TargetField(name='std::str')),
|
|
1462
|
-
PtrInfo(name='target', target=TargetField(name='std::str')),
|
|
1463
|
-
*[PtrInfo(**prop.dict()) for prop in field.properties]
|
|
1464
|
-
]
|
|
1465
|
-
)
|
|
1466
|
-
link_df = temp_structure.fit(link_df)
|
|
1467
|
-
link = link_df.groupby('source').apply(_format_link, link_name=name)
|
|
1468
|
-
data = data.drop(columns=[name], errors='ignore')
|
|
1469
|
-
data = data.join(link.to_frame(name), on=bkey)
|
|
1470
|
-
return data
|
|
1471
|
-
|
|
1472
1526
|
async def insert_df_pg(
|
|
1473
1527
|
self,
|
|
1474
1528
|
object_name: str,
|
|
@@ -1584,6 +1638,80 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1584
1638
|
finally:
|
|
1585
1639
|
await conn.close()
|
|
1586
1640
|
|
|
1641
|
+
@txn_support
|
|
1642
|
+
async def update_df(
|
|
1643
|
+
self,
|
|
1644
|
+
object_name: str,
|
|
1645
|
+
data: pd.DataFrame,
|
|
1646
|
+
relation: Dict[str, pd.DataFrame] = None,
|
|
1647
|
+
chunksize: int = 500,
|
|
1648
|
+
match_fields: Iterable[str] = None,
|
|
1649
|
+
) -> None:
|
|
1650
|
+
"""以事务执行基于DataFrame字段信息的批量更新数据
|
|
1651
|
+
|
|
1652
|
+
将以业务主键作为匹配条件,除业务主键以外的字段将为update fields
|
|
1653
|
+
|
|
1654
|
+
Args:
|
|
1655
|
+
object_name: 被插入数据的对象名,需属于当前应用
|
|
1656
|
+
data: 要插入的数据,若有single link property,
|
|
1657
|
+
则以列名为link_name@link_property_name的形式提供
|
|
1658
|
+
relation: 如有multi link,提供该字典用于补充link target信息,
|
|
1659
|
+
键为link字段名,值为映射关系的DataFrame
|
|
1660
|
+
DataFrame中的source列需为插入对象的业务主键,
|
|
1661
|
+
target列需为link target的业务主键,
|
|
1662
|
+
若有link property,则以property名为列名,提供在除source和target的列中
|
|
1663
|
+
chunksize: 单次最大行数
|
|
1664
|
+
match_fields: update的匹配列表,涉及的fields需出现在data或relation中,默认为业务主键
|
|
1665
|
+
"""
|
|
1666
|
+
if data.empty:
|
|
1667
|
+
logger.info("data为空,无DML执行")
|
|
1668
|
+
return
|
|
1669
|
+
|
|
1670
|
+
if object_name in self.objects:
|
|
1671
|
+
obj = self.objects[object_name]
|
|
1672
|
+
else:
|
|
1673
|
+
raise ObjectNotExist(
|
|
1674
|
+
f'DeepModel对象[{object_name}]在当前应用不存在,无法更新数据'
|
|
1675
|
+
)
|
|
1676
|
+
if obj.external:
|
|
1677
|
+
raise ExternalObjectReadOnly('外部对象只可读')
|
|
1678
|
+
|
|
1679
|
+
structure = ObjectStructure(name=obj.name, structure=obj.fields.values())
|
|
1680
|
+
self._valid_data(data, object_name, relation, structure)
|
|
1681
|
+
relation = relation or {}
|
|
1682
|
+
bkey = await self._get_bkey(obj)
|
|
1683
|
+
# data拼接relation df
|
|
1684
|
+
data = self._merge_relation(data, relation, structure, bkey)
|
|
1685
|
+
# 从data中分离出self-link更新信息
|
|
1686
|
+
data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
|
|
1687
|
+
field_info, tgt_main_field = await self._collect_bulk_field_info(
|
|
1688
|
+
object_name, structure, data, relation
|
|
1689
|
+
)
|
|
1690
|
+
field_names = set(map(lambda f: f.name, field_info))
|
|
1691
|
+
|
|
1692
|
+
if missing := (set(match_fields) - set(field_names)):
|
|
1693
|
+
raise ValueError(f"match fields: {missing} 不在提供的数据中")
|
|
1694
|
+
|
|
1695
|
+
match_fields = set(match_fields or {bkey}) & set(field_names)
|
|
1696
|
+
update_ql = bulk_update_by_fields(
|
|
1697
|
+
object_name, field_info, tgt_main_field,
|
|
1698
|
+
match_fields, field_names - match_fields
|
|
1699
|
+
)
|
|
1700
|
+
qls = []
|
|
1701
|
+
self._collect_qls(structure.fit(data), update_ql, chunksize, qls)
|
|
1702
|
+
if self_link_dfs:
|
|
1703
|
+
for update_field, (update_df, main_field) in self_link_dfs.items():
|
|
1704
|
+
field = structure.fields[update_field]
|
|
1705
|
+
update_ql = bulk_update_by_fields(
|
|
1706
|
+
object_name, [field],
|
|
1707
|
+
{update_field: main_field},
|
|
1708
|
+
[bkey],
|
|
1709
|
+
[update_field]
|
|
1710
|
+
)
|
|
1711
|
+
self._collect_qls(update_df, update_ql, chunksize, qls)
|
|
1712
|
+
|
|
1713
|
+
await self.execute(qls)
|
|
1714
|
+
|
|
1587
1715
|
@asynccontextmanager
|
|
1588
1716
|
async def start_transaction(self, flatten: bool = False):
|
|
1589
1717
|
"""开启事务
|
|
@@ -1661,11 +1789,14 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1661
1789
|
raise NotImplemented('非直连模式不支持设置state信息')
|
|
1662
1790
|
else:
|
|
1663
1791
|
bak_cli = self.client
|
|
1792
|
+
bak_globals = self.client._options.state._globals
|
|
1664
1793
|
try:
|
|
1665
1794
|
self.client = self.client.with_globals(**globals_)
|
|
1795
|
+
self._globals = self.client._options.state._globals
|
|
1666
1796
|
yield
|
|
1667
1797
|
finally:
|
|
1668
1798
|
self.client = bak_cli
|
|
1799
|
+
self._globals = bak_globals
|
|
1669
1800
|
|
|
1670
1801
|
@contextmanager
|
|
1671
1802
|
def without_globals(self, *global_names):
|
|
@@ -1676,18 +1807,22 @@ class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
|
1676
1807
|
raise NotImplemented('非直连模式不支持设置state信息')
|
|
1677
1808
|
else:
|
|
1678
1809
|
bak_cli = self.client
|
|
1810
|
+
bak_globals = self.client._options.state._globals
|
|
1679
1811
|
try:
|
|
1680
1812
|
self.client = self.client.without_globals(*global_names)
|
|
1813
|
+
self._globals = self.client._options.state._globals
|
|
1681
1814
|
yield
|
|
1682
1815
|
finally:
|
|
1683
1816
|
self.client = bak_cli
|
|
1817
|
+
self._globals = bak_globals
|
|
1684
1818
|
|
|
1685
1819
|
|
|
1686
1820
|
class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
|
|
1687
1821
|
synchronize = (
|
|
1688
1822
|
'query_object', 'query', 'query_df',
|
|
1689
1823
|
'execute', 'get_object',
|
|
1690
|
-
'insert_df', 'insert_df_pg'
|
|
1824
|
+
'insert_df', 'insert_df_pg',
|
|
1825
|
+
'update_df',
|
|
1691
1826
|
)
|
|
1692
1827
|
|
|
1693
1828
|
if TYPE_CHECKING: # pragma: no cover
|
|
@@ -1737,6 +1872,16 @@ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
|
|
|
1737
1872
|
) -> None:
|
|
1738
1873
|
...
|
|
1739
1874
|
|
|
1875
|
+
def update_df(
|
|
1876
|
+
self,
|
|
1877
|
+
object_name: str,
|
|
1878
|
+
data: pd.DataFrame,
|
|
1879
|
+
relation: Dict[str, pd.DataFrame] = None,
|
|
1880
|
+
chunksize: int = 500,
|
|
1881
|
+
match_fields: Iterable[str] = None,
|
|
1882
|
+
) -> None:
|
|
1883
|
+
...
|
|
1884
|
+
|
|
1740
1885
|
@contextmanager
|
|
1741
1886
|
def start_transaction(self, flatten: bool = False):
|
|
1742
1887
|
"""开启事务
|