deepfos 1.1.60__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepfos/__init__.py +6 -0
- deepfos/_version.py +21 -0
- deepfos/algo/__init__.py +0 -0
- deepfos/algo/graph.py +171 -0
- deepfos/algo/segtree.py +31 -0
- deepfos/api/V1_1/__init__.py +0 -0
- deepfos/api/V1_1/business_model.py +119 -0
- deepfos/api/V1_1/dimension.py +599 -0
- deepfos/api/V1_1/models/__init__.py +0 -0
- deepfos/api/V1_1/models/business_model.py +1033 -0
- deepfos/api/V1_1/models/dimension.py +2768 -0
- deepfos/api/V1_2/__init__.py +0 -0
- deepfos/api/V1_2/dimension.py +285 -0
- deepfos/api/V1_2/models/__init__.py +0 -0
- deepfos/api/V1_2/models/dimension.py +2923 -0
- deepfos/api/__init__.py +0 -0
- deepfos/api/account.py +167 -0
- deepfos/api/accounting_engines.py +147 -0
- deepfos/api/app.py +626 -0
- deepfos/api/approval_process.py +198 -0
- deepfos/api/base.py +983 -0
- deepfos/api/business_model.py +160 -0
- deepfos/api/consolidation.py +129 -0
- deepfos/api/consolidation_process.py +106 -0
- deepfos/api/datatable.py +341 -0
- deepfos/api/deep_pipeline.py +61 -0
- deepfos/api/deepconnector.py +36 -0
- deepfos/api/deepfos_task.py +92 -0
- deepfos/api/deepmodel.py +188 -0
- deepfos/api/dimension.py +486 -0
- deepfos/api/financial_model.py +319 -0
- deepfos/api/journal_model.py +119 -0
- deepfos/api/journal_template.py +132 -0
- deepfos/api/memory_financial_model.py +98 -0
- deepfos/api/models/__init__.py +3 -0
- deepfos/api/models/account.py +483 -0
- deepfos/api/models/accounting_engines.py +756 -0
- deepfos/api/models/app.py +1338 -0
- deepfos/api/models/approval_process.py +1043 -0
- deepfos/api/models/base.py +234 -0
- deepfos/api/models/business_model.py +805 -0
- deepfos/api/models/consolidation.py +711 -0
- deepfos/api/models/consolidation_process.py +248 -0
- deepfos/api/models/datatable_mysql.py +427 -0
- deepfos/api/models/deep_pipeline.py +55 -0
- deepfos/api/models/deepconnector.py +28 -0
- deepfos/api/models/deepfos_task.py +386 -0
- deepfos/api/models/deepmodel.py +308 -0
- deepfos/api/models/dimension.py +1576 -0
- deepfos/api/models/financial_model.py +1796 -0
- deepfos/api/models/journal_model.py +341 -0
- deepfos/api/models/journal_template.py +854 -0
- deepfos/api/models/memory_financial_model.py +478 -0
- deepfos/api/models/platform.py +178 -0
- deepfos/api/models/python.py +221 -0
- deepfos/api/models/reconciliation_engine.py +411 -0
- deepfos/api/models/reconciliation_report.py +161 -0
- deepfos/api/models/role_strategy.py +884 -0
- deepfos/api/models/smartlist.py +237 -0
- deepfos/api/models/space.py +1137 -0
- deepfos/api/models/system.py +1065 -0
- deepfos/api/models/variable.py +463 -0
- deepfos/api/models/workflow.py +946 -0
- deepfos/api/platform.py +199 -0
- deepfos/api/python.py +90 -0
- deepfos/api/reconciliation_engine.py +181 -0
- deepfos/api/reconciliation_report.py +64 -0
- deepfos/api/role_strategy.py +234 -0
- deepfos/api/smartlist.py +69 -0
- deepfos/api/space.py +582 -0
- deepfos/api/system.py +372 -0
- deepfos/api/variable.py +154 -0
- deepfos/api/workflow.py +264 -0
- deepfos/boost/__init__.py +6 -0
- deepfos/boost/py_jstream.py +89 -0
- deepfos/boost/py_pandas.py +20 -0
- deepfos/cache.py +121 -0
- deepfos/config.py +6 -0
- deepfos/core/__init__.py +27 -0
- deepfos/core/cube/__init__.py +10 -0
- deepfos/core/cube/_base.py +462 -0
- deepfos/core/cube/constants.py +21 -0
- deepfos/core/cube/cube.py +408 -0
- deepfos/core/cube/formula.py +707 -0
- deepfos/core/cube/syscube.py +532 -0
- deepfos/core/cube/typing.py +7 -0
- deepfos/core/cube/utils.py +238 -0
- deepfos/core/dimension/__init__.py +11 -0
- deepfos/core/dimension/_base.py +506 -0
- deepfos/core/dimension/dimcreator.py +184 -0
- deepfos/core/dimension/dimension.py +472 -0
- deepfos/core/dimension/dimexpr.py +271 -0
- deepfos/core/dimension/dimmember.py +155 -0
- deepfos/core/dimension/eledimension.py +22 -0
- deepfos/core/dimension/filters.py +99 -0
- deepfos/core/dimension/sysdimension.py +168 -0
- deepfos/core/logictable/__init__.py +5 -0
- deepfos/core/logictable/_cache.py +141 -0
- deepfos/core/logictable/_operator.py +663 -0
- deepfos/core/logictable/nodemixin.py +673 -0
- deepfos/core/logictable/sqlcondition.py +609 -0
- deepfos/core/logictable/tablemodel.py +497 -0
- deepfos/db/__init__.py +36 -0
- deepfos/db/cipher.py +660 -0
- deepfos/db/clickhouse.py +191 -0
- deepfos/db/connector.py +195 -0
- deepfos/db/daclickhouse.py +171 -0
- deepfos/db/dameng.py +101 -0
- deepfos/db/damysql.py +189 -0
- deepfos/db/dbkits.py +358 -0
- deepfos/db/deepengine.py +99 -0
- deepfos/db/deepmodel.py +82 -0
- deepfos/db/deepmodel_kingbase.py +83 -0
- deepfos/db/edb.py +214 -0
- deepfos/db/gauss.py +83 -0
- deepfos/db/kingbase.py +83 -0
- deepfos/db/mysql.py +184 -0
- deepfos/db/oracle.py +131 -0
- deepfos/db/postgresql.py +192 -0
- deepfos/db/sqlserver.py +99 -0
- deepfos/db/utils.py +135 -0
- deepfos/element/__init__.py +89 -0
- deepfos/element/accounting.py +348 -0
- deepfos/element/apvlprocess.py +215 -0
- deepfos/element/base.py +398 -0
- deepfos/element/bizmodel.py +1269 -0
- deepfos/element/datatable.py +2467 -0
- deepfos/element/deep_pipeline.py +186 -0
- deepfos/element/deepconnector.py +59 -0
- deepfos/element/deepmodel.py +1806 -0
- deepfos/element/dimension.py +1254 -0
- deepfos/element/fact_table.py +427 -0
- deepfos/element/finmodel.py +1485 -0
- deepfos/element/journal.py +840 -0
- deepfos/element/journal_template.py +943 -0
- deepfos/element/pyscript.py +412 -0
- deepfos/element/reconciliation.py +553 -0
- deepfos/element/rolestrategy.py +243 -0
- deepfos/element/smartlist.py +457 -0
- deepfos/element/variable.py +756 -0
- deepfos/element/workflow.py +560 -0
- deepfos/exceptions/__init__.py +239 -0
- deepfos/exceptions/hook.py +86 -0
- deepfos/lazy.py +104 -0
- deepfos/lazy_import.py +84 -0
- deepfos/lib/__init__.py +0 -0
- deepfos/lib/_javaobj.py +366 -0
- deepfos/lib/asynchronous.py +879 -0
- deepfos/lib/concurrency.py +107 -0
- deepfos/lib/constant.py +39 -0
- deepfos/lib/decorator.py +310 -0
- deepfos/lib/deepchart.py +778 -0
- deepfos/lib/deepux.py +477 -0
- deepfos/lib/discovery.py +273 -0
- deepfos/lib/edb_lexer.py +789 -0
- deepfos/lib/eureka.py +156 -0
- deepfos/lib/filterparser.py +751 -0
- deepfos/lib/httpcli.py +106 -0
- deepfos/lib/jsonstreamer.py +80 -0
- deepfos/lib/msg.py +394 -0
- deepfos/lib/nacos.py +225 -0
- deepfos/lib/patch.py +92 -0
- deepfos/lib/redis.py +241 -0
- deepfos/lib/serutils.py +181 -0
- deepfos/lib/stopwatch.py +99 -0
- deepfos/lib/subtask.py +572 -0
- deepfos/lib/sysutils.py +703 -0
- deepfos/lib/utils.py +1003 -0
- deepfos/local.py +160 -0
- deepfos/options.py +670 -0
- deepfos/translation.py +237 -0
- deepfos-1.1.60.dist-info/METADATA +33 -0
- deepfos-1.1.60.dist-info/RECORD +175 -0
- deepfos-1.1.60.dist-info/WHEEL +5 -0
- deepfos-1.1.60.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1806 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
from asyncpg.connection import connect as pg_conn
|
|
5
|
+
import json
|
|
6
|
+
import uuid
|
|
7
|
+
from contextlib import asynccontextmanager, contextmanager
|
|
8
|
+
from contextvars import ContextVar
|
|
9
|
+
from itertools import count
|
|
10
|
+
from typing import (
|
|
11
|
+
List, TYPE_CHECKING, Any, Dict, Union, NamedTuple,
|
|
12
|
+
Iterable, Optional, Literal
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
import edgedb
|
|
16
|
+
import pandas as pd
|
|
17
|
+
from loguru import logger
|
|
18
|
+
from pydantic import BaseModel, parse_obj_as, ValidationError, Field
|
|
19
|
+
|
|
20
|
+
from deepfos import OPTION
|
|
21
|
+
from deepfos.api.deepmodel import DeepModelAPI
|
|
22
|
+
from deepfos.api.models.deepmodel import (
|
|
23
|
+
ObjectBasicDTO, ObjectParam,
|
|
24
|
+
QueryResultObjectInfo, QueryResult
|
|
25
|
+
)
|
|
26
|
+
from deepfos.cache import SpaceSeperatedLRUCache
|
|
27
|
+
from deepfos.db.edb import create_async_client
|
|
28
|
+
from deepfos.element.base import ElementBase, SyncMeta
|
|
29
|
+
from deepfos.exceptions import (
|
|
30
|
+
RequiredFieldUnfilled, ObjectNotExist,
|
|
31
|
+
ExternalObjectReadOnly, MultiLinkTargetNotUnique,
|
|
32
|
+
SingleLinkInRelation
|
|
33
|
+
)
|
|
34
|
+
from deepfos.lib import serutils
|
|
35
|
+
from deepfos.lib.asynchronous import future_property, evloop
|
|
36
|
+
from deepfos.lib.decorator import flagmethod, cached_property, lru_cache
|
|
37
|
+
from deepfos.lib.utils import AliasGenerator, to_version_tuple
|
|
38
|
+
|
|
39
|
+
__all__ = ['AsyncDeepModel', 'DeepModel', 'to_fields', 'QueryWithArgs']
|
|
40
|
+
|
|
41
|
+
OBJECT_QUERY = \
|
|
42
|
+
"""
|
|
43
|
+
with module schema
|
|
44
|
+
select ObjectType {
|
|
45
|
+
id,
|
|
46
|
+
name,
|
|
47
|
+
links: {
|
|
48
|
+
id,
|
|
49
|
+
name,
|
|
50
|
+
cardinality,
|
|
51
|
+
required,
|
|
52
|
+
properties: { id, name, target: {name} } filter .name not in {'source', 'target'},
|
|
53
|
+
target: { id, name, external, annotations: {name, value := @value},
|
|
54
|
+
properties: { id, name } filter .name != 'id' },
|
|
55
|
+
expr,
|
|
56
|
+
constraints: { name, expr, finalexpr, subjectexpr },
|
|
57
|
+
source_property: {name},
|
|
58
|
+
target_property: {name},
|
|
59
|
+
annotations: {name, value := @value},
|
|
60
|
+
} filter .name != '__type__',
|
|
61
|
+
properties: {
|
|
62
|
+
id,
|
|
63
|
+
name,
|
|
64
|
+
cardinality,
|
|
65
|
+
required,
|
|
66
|
+
target: { name },
|
|
67
|
+
expr,
|
|
68
|
+
constraints: { name, expr, finalexpr, subjectexpr },
|
|
69
|
+
annotations: {name, value := @value},
|
|
70
|
+
} filter .name != 'id',
|
|
71
|
+
annotations: {name, value := @value},
|
|
72
|
+
constraints: { name, expr, finalexpr, subjectexpr },
|
|
73
|
+
external
|
|
74
|
+
}
|
|
75
|
+
"""
|
|
76
|
+
BUSINESS_KEY = 'business_key'
|
|
77
|
+
BATCH_INSERT_KW = 'data'
|
|
78
|
+
|
|
79
|
+
# pandas to_json精度最大为15
|
|
80
|
+
# 为避免精度缺失,如下类型需转换为string
|
|
81
|
+
NEED_CAST_STR = ['std::decimal']
|
|
82
|
+
|
|
83
|
+
DOC_ARGS_KWARGS = """
|
|
84
|
+
Hint:
|
|
85
|
+
|
|
86
|
+
kwargs语法:
|
|
87
|
+
|
|
88
|
+
select User{name, is_active}
|
|
89
|
+
filter .name=<std::str>$name and is_active=<std::bool>$active
|
|
90
|
+
|
|
91
|
+
.. admonition:: 使用示例
|
|
92
|
+
|
|
93
|
+
.. code-block:: python
|
|
94
|
+
|
|
95
|
+
dm = DeepModel()
|
|
96
|
+
|
|
97
|
+
dm.execute(
|
|
98
|
+
'''delete User filter .name=<std::str>$name
|
|
99
|
+
and is_active=<std::bool>$active''',
|
|
100
|
+
name='Alice', active='True'
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
此处 `$` 为以kwargs的方式指定参数的特殊符号,
|
|
104
|
+
且需在参数前增加相应类型提示,参数值只支持str和int类型
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
NOT_SCALAR = "default::object"
|
|
108
|
+
|
|
109
|
+
dm_type_to_edb_scalar = {
|
|
110
|
+
'str': 'std::str',
|
|
111
|
+
'int': 'std::int64',
|
|
112
|
+
'bool': 'std::bool',
|
|
113
|
+
'multilingual': 'std::str',
|
|
114
|
+
'float': 'std::decimal',
|
|
115
|
+
'datetime': 'cal::local_datetime',
|
|
116
|
+
'file': 'std::str',
|
|
117
|
+
'uuid': 'std::str',
|
|
118
|
+
'json': 'std::json',
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class ObjectElement(ObjectParam):
|
|
123
|
+
@property
|
|
124
|
+
def links(self):
|
|
125
|
+
return {link.code: link for link in self.linkParamList}
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class QueryWithArgs(BaseModel):
|
|
129
|
+
commands: str
|
|
130
|
+
kwargs: Dict[str, Any] = Field(default_factory=dict)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class MainField(NamedTuple):
|
|
134
|
+
business_key: str
|
|
135
|
+
is_multi: bool
|
|
136
|
+
props: Iterable[Optional[str]]
|
|
137
|
+
# 目前业务主键创建的类型只会为std::str
|
|
138
|
+
type: str = 'std::str'
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class MainPgCol(NamedTuple):
|
|
142
|
+
target_bkey_col: str
|
|
143
|
+
target_col: str
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class ConstraintField(BaseModel):
|
|
147
|
+
name: str
|
|
148
|
+
expr: str
|
|
149
|
+
finalexpr: str = None
|
|
150
|
+
subjectexpr: str = None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class NamedField(BaseModel):
|
|
154
|
+
name: str = None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class NameIdField(BaseModel):
|
|
158
|
+
id: uuid.UUID = None
|
|
159
|
+
name: str = None
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class TargetField(BaseModel):
|
|
163
|
+
id: uuid.UUID = None
|
|
164
|
+
name: str
|
|
165
|
+
external: bool = False
|
|
166
|
+
annotations: List[Dict[str, str]] = Field(default_factory=list)
|
|
167
|
+
properties: List[NameIdField] = None
|
|
168
|
+
|
|
169
|
+
@property
|
|
170
|
+
def is_scalar(self) -> bool:
|
|
171
|
+
return self.name.startswith('std::') or self.name == 'cal::local_datetime'
|
|
172
|
+
|
|
173
|
+
@property
|
|
174
|
+
def info(self):
|
|
175
|
+
return {e['name'].rpartition('::')[-1]: e['value'] for e in self.annotations}
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def normalized_name(self):
|
|
179
|
+
return self.name.rpartition('::')[-1]
|
|
180
|
+
|
|
181
|
+
@property
|
|
182
|
+
def props(self):
|
|
183
|
+
return {p.name: p for p in self.properties}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class LinkPropField(BaseModel):
|
|
187
|
+
id: uuid.UUID = None
|
|
188
|
+
name: str
|
|
189
|
+
target: NamedField
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def type(self) -> str:
|
|
193
|
+
return self.target.name
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class PtrInfo(BaseModel):
|
|
197
|
+
id: Union[uuid.UUID, Literal['id']] = None
|
|
198
|
+
name: str
|
|
199
|
+
target: TargetField
|
|
200
|
+
properties: List[LinkPropField] = Field(default_factory=list)
|
|
201
|
+
expr: str = None
|
|
202
|
+
source_property: NamedField = None
|
|
203
|
+
target_property: NamedField = None
|
|
204
|
+
required: bool = False
|
|
205
|
+
cardinality: str = None
|
|
206
|
+
constraints: List[ConstraintField] = Field(default_factory=list)
|
|
207
|
+
annotations: List[Dict[str, str]] = Field(default_factory=list)
|
|
208
|
+
|
|
209
|
+
@property
|
|
210
|
+
def type(self) -> str:
|
|
211
|
+
return self.target.name
|
|
212
|
+
|
|
213
|
+
@property
|
|
214
|
+
def is_link(self):
|
|
215
|
+
return not self.target.is_scalar
|
|
216
|
+
|
|
217
|
+
@property
|
|
218
|
+
def is_multi_link(self):
|
|
219
|
+
return self.is_link and self.cardinality == 'Many'
|
|
220
|
+
|
|
221
|
+
@property
|
|
222
|
+
def is_multi(self):
|
|
223
|
+
return self.cardinality == 'Many'
|
|
224
|
+
|
|
225
|
+
@property
|
|
226
|
+
def computable(self):
|
|
227
|
+
return self.expr is not None
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def external(self):
|
|
231
|
+
return self.target.external
|
|
232
|
+
|
|
233
|
+
@property
|
|
234
|
+
def props(self):
|
|
235
|
+
return [p.name for p in self.properties]
|
|
236
|
+
|
|
237
|
+
@property
|
|
238
|
+
def prop_type(self):
|
|
239
|
+
return {p.name: p.type for p in self.properties}
|
|
240
|
+
|
|
241
|
+
@property
|
|
242
|
+
def target_col(self):
|
|
243
|
+
if self.target_property:
|
|
244
|
+
return self.target_property.name
|
|
245
|
+
|
|
246
|
+
@property
|
|
247
|
+
def info(self):
|
|
248
|
+
return {e['name'].rpartition('::')[-1]: e['value'] for e in self.annotations}
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
_RE_CONSTRAINT_FIELDS = re.compile(r'\((((\.\w+)(,\s+)?)+)\)')
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
class ObjectTypeFrame(BaseModel):
|
|
255
|
+
id: uuid.UUID = None
|
|
256
|
+
name: str
|
|
257
|
+
links: List[PtrInfo] = Field(default_factory=list)
|
|
258
|
+
properties: List[PtrInfo] = Field(default_factory=list)
|
|
259
|
+
external: bool
|
|
260
|
+
annotations: List[Dict[str, str]] = Field(default_factory=list)
|
|
261
|
+
constraints: List[ConstraintField] = Field(default_factory=list)
|
|
262
|
+
|
|
263
|
+
@property
|
|
264
|
+
def fields(self):
|
|
265
|
+
return {ptr.name: ptr for ptr in [*self.links, *self.properties]}
|
|
266
|
+
|
|
267
|
+
@property
|
|
268
|
+
def info(self):
|
|
269
|
+
return {e['name'].rpartition('::')[-1]: e['value'] for e in self.annotations}
|
|
270
|
+
|
|
271
|
+
@property
|
|
272
|
+
def normalized_name(self):
|
|
273
|
+
return self.name.rpartition('::')[-1]
|
|
274
|
+
|
|
275
|
+
@property
|
|
276
|
+
def exclusive_fields(self):
|
|
277
|
+
exclusive = [
|
|
278
|
+
field.name
|
|
279
|
+
for field in self.fields.values()
|
|
280
|
+
if field.name != 'id' and any([
|
|
281
|
+
const.expr and (
|
|
282
|
+
const.name == 'std::exclusive'
|
|
283
|
+
and const.subjectexpr is None
|
|
284
|
+
)
|
|
285
|
+
for const in field.constraints
|
|
286
|
+
])
|
|
287
|
+
]
|
|
288
|
+
for const in self.constraints:
|
|
289
|
+
if (
|
|
290
|
+
const.name == 'std::exclusive'
|
|
291
|
+
and const.subjectexpr and (
|
|
292
|
+
m := _RE_CONSTRAINT_FIELDS.match(const.subjectexpr)
|
|
293
|
+
)
|
|
294
|
+
):
|
|
295
|
+
found = m.group(1)
|
|
296
|
+
exclusive.append({f.strip()[1:] for f in found.split(',')})
|
|
297
|
+
return exclusive
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _format_link(df: pd.DataFrame, link_name: str):
|
|
301
|
+
if all(pd.isnull(df['target'])):
|
|
302
|
+
return {'target': pd.NA}
|
|
303
|
+
|
|
304
|
+
record = df.drop(columns=['source']).set_index('target')
|
|
305
|
+
|
|
306
|
+
if not record.index.is_unique:
|
|
307
|
+
raise MultiLinkTargetNotUnique(
|
|
308
|
+
f'Multi Link: [{link_name}] relation dataframe中'
|
|
309
|
+
f'source与target对应存在不唯一性'
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
record = record.to_dict(orient='index')
|
|
313
|
+
return {'prop': record, 'target': list(record.keys())}
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
class BaseField(PtrInfo):
|
|
317
|
+
def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
|
|
318
|
+
"""使 :class:`Dataframe` 对应的列符合字段的限制条件
|
|
319
|
+
|
|
320
|
+
Args:
|
|
321
|
+
df: 待转换的 :class:`Dataframe`
|
|
322
|
+
field_name: 需要转化的列名
|
|
323
|
+
raw_pg: 是否在为pg插入做fit
|
|
324
|
+
"""
|
|
325
|
+
pass
|
|
326
|
+
|
|
327
|
+
def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
|
|
328
|
+
"""
|
|
329
|
+
对 :class:`Dataframe` 对应的列作类型转换。
|
|
330
|
+
一般在获取 :class:`Dataframe` 时使用。
|
|
331
|
+
"""
|
|
332
|
+
pass
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class FieldDateTime(BaseField):
|
|
336
|
+
@staticmethod
|
|
337
|
+
def format_datetime(dt):
|
|
338
|
+
if pd.isna(dt):
|
|
339
|
+
return pd.NA
|
|
340
|
+
return pd.to_datetime(dt).strftime("%Y-%m-%dT%H:%M:%S")
|
|
341
|
+
|
|
342
|
+
def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
|
|
343
|
+
df[field_name] = df[field_name].apply(self.format_datetime)
|
|
344
|
+
|
|
345
|
+
def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
|
|
346
|
+
df[field_name] = pd.to_datetime(df[field_name], errors='ignore')
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
class FieldString(BaseField):
|
|
350
|
+
def format_string(self, data):
|
|
351
|
+
if pd.isna(data):
|
|
352
|
+
return pd.NA
|
|
353
|
+
return str(data)
|
|
354
|
+
|
|
355
|
+
def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
|
|
356
|
+
df[field_name] = df[field_name].apply(self.format_string)
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
class FieldJson(BaseField):
|
|
360
|
+
def format_json(self, data):
|
|
361
|
+
if pd.isna(data):
|
|
362
|
+
return data
|
|
363
|
+
return json.dumps(data)
|
|
364
|
+
|
|
365
|
+
def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
|
|
366
|
+
# std::json needed to be fit only when data is for pg batch insert
|
|
367
|
+
# because json-object (dict in pd series) will be encoded correctly
|
|
368
|
+
# by pandas to_json
|
|
369
|
+
if raw_pg:
|
|
370
|
+
df[field_name] = df[field_name].apply(self.format_json)
|
|
371
|
+
|
|
372
|
+
def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
|
|
373
|
+
# std::json needed to be cast only when data is from http
|
|
374
|
+
# since json value will be converted to json string(type: str)
|
|
375
|
+
# in edgedb python protocol
|
|
376
|
+
if not direct_access:
|
|
377
|
+
df[field_name] = df[field_name].apply(self.format_json)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
class FieldInt(BaseField):
|
|
381
|
+
def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
|
|
382
|
+
df[field_name] = pd.to_numeric(df[field_name])
|
|
383
|
+
|
|
384
|
+
def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
|
|
385
|
+
df[field_name] = df[field_name].astype(pd.Int64Dtype(), errors='ignore')
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
class FieldDecimal(FieldString):
|
|
389
|
+
def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
|
|
390
|
+
df[field_name] = df[field_name].astype(float, errors='ignore')
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
class FieldBool(BaseField):
|
|
394
|
+
def format_bool(self, data):
|
|
395
|
+
if isinstance(data, bool) or pd.isna(data):
|
|
396
|
+
return data
|
|
397
|
+
return str(data).lower() == 'true'
|
|
398
|
+
|
|
399
|
+
def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
|
|
400
|
+
df[field_name] = df[field_name].apply(self.format_bool)
|
|
401
|
+
|
|
402
|
+
def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
|
|
403
|
+
pass
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
class FieldFactory:
|
|
407
|
+
field_map = {
|
|
408
|
+
'std::bool': FieldBool,
|
|
409
|
+
'std::int64': FieldInt,
|
|
410
|
+
'std::bigint': FieldInt,
|
|
411
|
+
'std::decimal': FieldDecimal,
|
|
412
|
+
'std::json': FieldJson,
|
|
413
|
+
'std::str': FieldString,
|
|
414
|
+
'cal::local_datetime': FieldDateTime,
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
def __new__(cls, field: Union[PtrInfo, LinkPropField]) -> BaseField:
|
|
418
|
+
field_class = cls.field_map.get(field.type, BaseField)
|
|
419
|
+
return field_class(**field.dict())
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
class ObjectStructure:
|
|
423
|
+
fields: Dict[str, BaseField]
|
|
424
|
+
|
|
425
|
+
def __init__(self, name, structure: Iterable[PtrInfo], include_id=False):
|
|
426
|
+
self.name = name
|
|
427
|
+
self.fields = {
|
|
428
|
+
field.name: FieldFactory(field)
|
|
429
|
+
for field in structure
|
|
430
|
+
if not field.computable and field.name != 'id'
|
|
431
|
+
}
|
|
432
|
+
# Make one for batch insert pg
|
|
433
|
+
if include_id:
|
|
434
|
+
self.fields['id'] = BaseField(
|
|
435
|
+
name='id', id='id', cardinality='One',
|
|
436
|
+
target=TargetField(name='std::uuid')
|
|
437
|
+
)
|
|
438
|
+
self.self_link_fields = []
|
|
439
|
+
for name, field in list(self.fields.items()):
|
|
440
|
+
if field.type == self.name:
|
|
441
|
+
self.self_link_fields.append(name)
|
|
442
|
+
if field.is_multi_link:
|
|
443
|
+
continue
|
|
444
|
+
if not field.is_link:
|
|
445
|
+
continue
|
|
446
|
+
for prop in field.properties:
|
|
447
|
+
self.fields[f'{name}@{prop.name}'] = FieldFactory(prop)
|
|
448
|
+
|
|
449
|
+
def fit(self, df: pd.DataFrame, raw_pg=False) -> pd.DataFrame:
|
|
450
|
+
"""
|
|
451
|
+
对传入的DataFrame的指定数据列执行fit操作。
|
|
452
|
+
直接影响DataFrame数据。
|
|
453
|
+
|
|
454
|
+
Args:
|
|
455
|
+
df: 数据源
|
|
456
|
+
raw_pg: 是否在为pg插入做fit
|
|
457
|
+
|
|
458
|
+
"""
|
|
459
|
+
valid_fields = []
|
|
460
|
+
for field in df.columns:
|
|
461
|
+
if field in self.fields:
|
|
462
|
+
valid_fields.append(field)
|
|
463
|
+
self.fields[field].fit(df, field, raw_pg)
|
|
464
|
+
|
|
465
|
+
return df[valid_fields]
|
|
466
|
+
|
|
467
|
+
def cast(self, df: pd.DataFrame, direct_access: bool = True):
|
|
468
|
+
for field in df.columns:
|
|
469
|
+
if field in self.fields:
|
|
470
|
+
self.fields[field].cast(df, field, direct_access)
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _iter_link_prop_assign(link, business_key, prop_name, prop_type, is_multi):
|
|
474
|
+
assign_string = f"@{prop_name} := <{prop_type}>"
|
|
475
|
+
if prop_type in NEED_CAST_STR:
|
|
476
|
+
assign_string += '<std::str>'
|
|
477
|
+
if is_multi:
|
|
478
|
+
return f"{assign_string}(json_get(item, '{link}', 'prop', .{business_key}, '{prop_name}'))"
|
|
479
|
+
|
|
480
|
+
return f"{assign_string}(json_get(item, '{link}@{prop_name}'))"
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _iter_single_assign(
|
|
484
|
+
field: PtrInfo,
|
|
485
|
+
cast_type: str,
|
|
486
|
+
target_main_field: Dict[str, MainField]
|
|
487
|
+
):
|
|
488
|
+
assign_string = f"{field.name} := "
|
|
489
|
+
# 设置标量值
|
|
490
|
+
if field.name not in target_main_field:
|
|
491
|
+
assign_string += f"<{cast_type}>"
|
|
492
|
+
|
|
493
|
+
if cast_type in NEED_CAST_STR:
|
|
494
|
+
assign_string += '<std::str>'
|
|
495
|
+
|
|
496
|
+
return assign_string + f"item['{field.name}']"
|
|
497
|
+
|
|
498
|
+
# 设置link target值
|
|
499
|
+
link = field.name
|
|
500
|
+
main_field = target_main_field[link]
|
|
501
|
+
|
|
502
|
+
if main_field.props:
|
|
503
|
+
target = (
|
|
504
|
+
cast_type + "{" +
|
|
505
|
+
",".join(
|
|
506
|
+
_iter_link_prop_assign(link, main_field.business_key, name,
|
|
507
|
+
field.prop_type[name], main_field.is_multi)
|
|
508
|
+
for name in main_field.props
|
|
509
|
+
) + "}"
|
|
510
|
+
)
|
|
511
|
+
else:
|
|
512
|
+
target = cast_type
|
|
513
|
+
|
|
514
|
+
if main_field.is_multi:
|
|
515
|
+
assign_string += f"""(
|
|
516
|
+
select detached {target}
|
|
517
|
+
filter contains(
|
|
518
|
+
<array<{main_field.type}>>(json_get(item, '{link}', 'target')),
|
|
519
|
+
.{main_field.business_key}
|
|
520
|
+
)
|
|
521
|
+
)"""
|
|
522
|
+
else:
|
|
523
|
+
assign_string += f"""assert_single((
|
|
524
|
+
select detached {target}
|
|
525
|
+
filter .{main_field.business_key} = <{main_field.type}>(json_get(item, '{link}'))
|
|
526
|
+
))"""
|
|
527
|
+
|
|
528
|
+
return assign_string
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def bulk_insert_by_fields(
|
|
532
|
+
object_name: str,
|
|
533
|
+
field_type: List[PtrInfo],
|
|
534
|
+
target_main_field: Dict[str, MainField],
|
|
535
|
+
):
|
|
536
|
+
insert_assign_body = ','.join(
|
|
537
|
+
[
|
|
538
|
+
_iter_single_assign(field, field.type, target_main_field)
|
|
539
|
+
for field in field_type
|
|
540
|
+
]
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
return f"""
|
|
544
|
+
with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
|
|
545
|
+
for item in json_array_unpack(raw_data) union (
|
|
546
|
+
insert {object_name} {{
|
|
547
|
+
{insert_assign_body}
|
|
548
|
+
}}
|
|
549
|
+
)
|
|
550
|
+
"""
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def bulk_upsert_by_fields(
|
|
554
|
+
object_name: str,
|
|
555
|
+
field_type: List[PtrInfo],
|
|
556
|
+
target_main_field: Dict[str, MainField],
|
|
557
|
+
exclusive_fields: Iterable[str],
|
|
558
|
+
update_fields: Iterable[str]
|
|
559
|
+
):
|
|
560
|
+
conflict_on_fields = map(lambda n: f'.{n}', exclusive_fields)
|
|
561
|
+
|
|
562
|
+
insert_assign_body = ','.join(
|
|
563
|
+
[
|
|
564
|
+
_iter_single_assign(field, field.type, target_main_field)
|
|
565
|
+
for field in field_type
|
|
566
|
+
]
|
|
567
|
+
)
|
|
568
|
+
update_assign_body = ','.join(
|
|
569
|
+
[
|
|
570
|
+
_iter_single_assign(field, field.type, target_main_field)
|
|
571
|
+
for field in field_type if field.name in update_fields
|
|
572
|
+
]
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
return f"""
|
|
576
|
+
with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
|
|
577
|
+
for item in json_array_unpack(raw_data) union (
|
|
578
|
+
insert {object_name} {{
|
|
579
|
+
{insert_assign_body}
|
|
580
|
+
}}
|
|
581
|
+
unless conflict on ({','.join(conflict_on_fields)})
|
|
582
|
+
else (
|
|
583
|
+
update {object_name} set {{
|
|
584
|
+
{update_assign_body}
|
|
585
|
+
}}
|
|
586
|
+
)
|
|
587
|
+
)
|
|
588
|
+
"""
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
def bulk_update_by_fields(
|
|
592
|
+
object_name: str,
|
|
593
|
+
business_key: str,
|
|
594
|
+
field_type: List[PtrInfo],
|
|
595
|
+
target_main_field: Dict[str, MainField],
|
|
596
|
+
update_fields: Iterable[str] = None,
|
|
597
|
+
):
|
|
598
|
+
update_assign_body = ','.join(
|
|
599
|
+
[
|
|
600
|
+
_iter_single_assign(field, field.type, target_main_field)
|
|
601
|
+
for field in field_type if field.name in update_fields
|
|
602
|
+
]
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
return f"""
|
|
606
|
+
with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
|
|
607
|
+
for item in json_array_unpack(raw_data) union (
|
|
608
|
+
update {object_name}
|
|
609
|
+
filter .{business_key} = <std::str>item['{business_key}']
|
|
610
|
+
set {{
|
|
611
|
+
{update_assign_body}
|
|
612
|
+
}}
|
|
613
|
+
)
|
|
614
|
+
"""
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def format_obj(obj: edgedb.Object) -> ObjectTypeFrame:
|
|
618
|
+
if not isinstance(obj, edgedb.Object):
|
|
619
|
+
raise TypeError("预期obj为edgedb.Object")
|
|
620
|
+
|
|
621
|
+
serialized = serutils.serialize(obj)
|
|
622
|
+
|
|
623
|
+
try:
|
|
624
|
+
return parse_obj_as(ObjectTypeFrame, serialized)
|
|
625
|
+
except ValidationError:
|
|
626
|
+
raise TypeError("预期obj为ObjectType查询得到的结构信息")
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def to_fields(obj: edgedb.Object) -> Dict[str, PtrInfo]:
|
|
630
|
+
return format_obj(obj).fields
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def collect_query_result_structure(
|
|
634
|
+
object_info: QueryResultObjectInfo
|
|
635
|
+
):
|
|
636
|
+
fields = [
|
|
637
|
+
PtrInfo(
|
|
638
|
+
name=f.name,
|
|
639
|
+
target=NamedField(name=dm_type_to_edb_scalar.get(f.type, NOT_SCALAR))
|
|
640
|
+
)
|
|
641
|
+
for f in object_info.fields
|
|
642
|
+
]
|
|
643
|
+
return ObjectStructure(name='', structure=fields)
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def collect_frame_desc_structure(desc: Dict[str, str]):
|
|
647
|
+
fields = [
|
|
648
|
+
PtrInfo(
|
|
649
|
+
name=name if isinstance(name, str) else name[0],
|
|
650
|
+
target=NamedField(
|
|
651
|
+
name=tname
|
|
652
|
+
if isinstance(tname, str) else NOT_SCALAR
|
|
653
|
+
)
|
|
654
|
+
)
|
|
655
|
+
for name, tname in desc.items()
|
|
656
|
+
]
|
|
657
|
+
return ObjectStructure(name='', structure=fields)
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
# Copied from edb/pgsql/types.py base_type_name_map
|
|
661
|
+
base_type_name_map = {
|
|
662
|
+
'std::str': ('text',),
|
|
663
|
+
'std::int64': ('int8',),
|
|
664
|
+
'std::decimal': ('numeric',),
|
|
665
|
+
'std::bool': ('bool',),
|
|
666
|
+
# for id col only
|
|
667
|
+
'std::uuid': ('uuid',),
|
|
668
|
+
'std::json': ('jsonb',),
|
|
669
|
+
'cal::local_datetime': ('edgedb', 'timestamp_t')
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def quote_ident(string: Union[str, uuid.UUID]) -> str:
|
|
674
|
+
return '"' + str(string).replace('"', '""') + '"'
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def pg_type_cast(edb_type: str):
|
|
678
|
+
if edb_type in base_type_name_map:
|
|
679
|
+
return '::' + '.'.join(base_type_name_map[edb_type])
|
|
680
|
+
return ''
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _iter_value(
|
|
684
|
+
field: PtrInfo,
|
|
685
|
+
idx: int,
|
|
686
|
+
target_info: Dict[str, MainPgCol],
|
|
687
|
+
alias: AliasGenerator
|
|
688
|
+
):
|
|
689
|
+
if field.name not in target_info:
|
|
690
|
+
if field.type == 'std::decimal':
|
|
691
|
+
return f'/*{field.name}*/"edgedb"."str_to_decimal"(${idx})'
|
|
692
|
+
if field.type == 'cal::local_datetime':
|
|
693
|
+
return f'/*{field.name}*/"edgedb"."local_datetime_in"(${idx})'
|
|
694
|
+
return f'/*{field.name}*/${idx}{pg_type_cast(field.type)}'
|
|
695
|
+
|
|
696
|
+
main_col = target_info[field.name]
|
|
697
|
+
tmp_table = quote_ident(alias.get('t'))
|
|
698
|
+
bkey = quote_ident(main_col.target_bkey_col)
|
|
699
|
+
# bkey is std::str
|
|
700
|
+
return (f"/*{field.name}*/\n"
|
|
701
|
+
f"(select {tmp_table}.{quote_ident(main_col.target_col)} "
|
|
702
|
+
f"from edgedbpub.{quote_ident(field.target.id)} as {tmp_table} "
|
|
703
|
+
f"where {tmp_table}.{bkey} = ${idx}{pg_type_cast('std::str')})")
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def batch_insert_pg(
|
|
707
|
+
object_id: str,
|
|
708
|
+
fields: List[PtrInfo],
|
|
709
|
+
target_info: Dict[str, MainPgCol],
|
|
710
|
+
exclusive_fields: List[PtrInfo] = None,
|
|
711
|
+
update_fields: List[PtrInfo] = None
|
|
712
|
+
):
|
|
713
|
+
upd_fields = list(map(lambda f: quote_ident(f.id), update_fields or []))
|
|
714
|
+
|
|
715
|
+
alias = AliasGenerator()
|
|
716
|
+
idx = count(1).__next__
|
|
717
|
+
col_assign = {}
|
|
718
|
+
for field in fields:
|
|
719
|
+
col = quote_ident(field.id)
|
|
720
|
+
col_assign[col] = _iter_value(field, idx(), target_info, alias)
|
|
721
|
+
|
|
722
|
+
insert_main_tbl = (
|
|
723
|
+
f'INSERT INTO edgedbpub.{quote_ident(object_id)}'
|
|
724
|
+
f'({", ".join(col_assign.keys())})'
|
|
725
|
+
f' VALUES ({", ".join(col_assign.values())})'
|
|
726
|
+
)
|
|
727
|
+
if exclusive_fields and upd_fields:
|
|
728
|
+
conflicts = ','.join(quote_ident(ef.id) for ef in exclusive_fields)
|
|
729
|
+
insert_main_tbl += (
|
|
730
|
+
f' ON CONFLICT ({conflicts}) DO UPDATE SET ' +
|
|
731
|
+
','.join([f'{col} = EXCLUDED.{col}' for col in upd_fields])
|
|
732
|
+
)
|
|
733
|
+
return insert_main_tbl
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
txn_support = flagmethod('_txn_support_')
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
class _TxnConfig:
|
|
740
|
+
__slots__ = ('qls', 'in_txn', 'txn_support', 'flatten')
|
|
741
|
+
|
|
742
|
+
def __init__(self):
|
|
743
|
+
self.qls = [[]]
|
|
744
|
+
self.in_txn = [False]
|
|
745
|
+
self.txn_support = False
|
|
746
|
+
self.flatten = False
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
@lru_cache(maxsize=128, cache_factory=SpaceSeperatedLRUCache)
|
|
750
|
+
async def get_element_info():
|
|
751
|
+
from deepfos.api.space import SpaceAPI
|
|
752
|
+
from deepfos.api.models.app import ElementRelationInfo
|
|
753
|
+
from deepfos.exceptions import ElementTypeMissingError
|
|
754
|
+
modules = await SpaceAPI(sync=False).module.get_usable_module()
|
|
755
|
+
target_module = ['MAINVIEW', 'DM']
|
|
756
|
+
for mdl in modules:
|
|
757
|
+
if mdl.moduleType in target_module and mdl.status == 1:
|
|
758
|
+
return ElementRelationInfo.construct_from(mdl)
|
|
759
|
+
raise ElementTypeMissingError('DeepModel组件在空间内不可用')
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
# -----------------------------------------------------------------------------
|
|
763
|
+
# core
|
|
764
|
+
class AsyncDeepModel(ElementBase[DeepModelAPI]):
|
|
765
|
+
"""DeepModel元素
|
|
766
|
+
|
|
767
|
+
Args:
|
|
768
|
+
direct_access: 是否使用直连模式
|
|
769
|
+
pg_dsn: PG连接信息
|
|
770
|
+
"""
|
|
771
|
+
__mangle_docs__ = False
|
|
772
|
+
|
|
773
|
+
def __init__(self, direct_access: bool = True, pg_dsn: str = None): # noqa
|
|
774
|
+
self._txn_ = ContextVar('QLTXN')
|
|
775
|
+
self.appmodule = f"app{OPTION.api.header['app']}"
|
|
776
|
+
self.spacemodule = f"space{OPTION.api.header['space']}"
|
|
777
|
+
self.direct_access = direct_access
|
|
778
|
+
self.alias = AliasGenerator()
|
|
779
|
+
self.pg_dsn = pg_dsn
|
|
780
|
+
|
|
781
|
+
@future_property
|
|
782
|
+
async def client(self):
|
|
783
|
+
if self.direct_access:
|
|
784
|
+
api = await self.wait_for('async_api')
|
|
785
|
+
ver = await api.extra.version()
|
|
786
|
+
if to_version_tuple(ver, -1) >= (3, 0, 18, 8, 0):
|
|
787
|
+
db_info = await api.sharding.database()
|
|
788
|
+
space = OPTION.api.header['space']
|
|
789
|
+
if db_info.space != space:
|
|
790
|
+
raise ValueError(
|
|
791
|
+
f'Space id in sharding database info invalid. '
|
|
792
|
+
f'Expected space id: {space}, actual: {db_info.space}'
|
|
793
|
+
)
|
|
794
|
+
dbname = db_info.edgedbName
|
|
795
|
+
else:
|
|
796
|
+
dbname = None
|
|
797
|
+
client = create_async_client(
|
|
798
|
+
default_module=self.appmodule,
|
|
799
|
+
dbname=dbname
|
|
800
|
+
)
|
|
801
|
+
if user_id := OPTION.api.header.get('user'):
|
|
802
|
+
client = client.with_globals(
|
|
803
|
+
**{
|
|
804
|
+
f'{self.spacemodule}::current_user_id':
|
|
805
|
+
user_id
|
|
806
|
+
}
|
|
807
|
+
)
|
|
808
|
+
return client
|
|
809
|
+
|
|
810
|
+
@future_property
|
|
811
|
+
async def element_info(self):
|
|
812
|
+
"""元素信息"""
|
|
813
|
+
return await get_element_info()
|
|
814
|
+
|
|
815
|
+
@future_property
|
|
816
|
+
async def async_api(self):
|
|
817
|
+
"""异步API对象"""
|
|
818
|
+
return await self._init_api()
|
|
819
|
+
|
|
820
|
+
def _safe_get_txn_conf(self) -> _TxnConfig:
|
|
821
|
+
try:
|
|
822
|
+
config = self._txn_.get()
|
|
823
|
+
except LookupError:
|
|
824
|
+
config = _TxnConfig()
|
|
825
|
+
self._txn_.set(config)
|
|
826
|
+
return config
|
|
827
|
+
|
|
828
|
+
@property
|
|
829
|
+
def _txn_support_(self):
|
|
830
|
+
return self._safe_get_txn_conf().txn_support
|
|
831
|
+
|
|
832
|
+
@_txn_support_.setter
|
|
833
|
+
def _txn_support_(self, val):
|
|
834
|
+
self._safe_get_txn_conf().txn_support = val
|
|
835
|
+
|
|
836
|
+
@future_property(on_demand=True)
|
|
837
|
+
async def model_objects(self) -> Dict[str, ObjectParam]:
|
|
838
|
+
"""MainView中的所有对象详情"""
|
|
839
|
+
api = await self.wait_for('async_api')
|
|
840
|
+
res = await api.object.get_all()
|
|
841
|
+
return {obj.code: obj for obj in res.objectList}
|
|
842
|
+
|
|
843
|
+
@future_property(on_demand=True)
|
|
844
|
+
async def model_object_list(self) -> List[ObjectBasicDTO]:
|
|
845
|
+
"""MainView中的所有对象列表"""
|
|
846
|
+
api = await self.wait_for('async_api')
|
|
847
|
+
return await api.object.list()
|
|
848
|
+
|
|
849
|
+
@future_property(on_demand=True)
|
|
850
|
+
async def user_objects(self) -> Dict[str, edgedb.Object]:
|
|
851
|
+
"""当前app下所有的用户对象"""
|
|
852
|
+
objects = await AsyncDeepModel.query_object(
|
|
853
|
+
self,
|
|
854
|
+
f"{OBJECT_QUERY} filter .name like '{self.appmodule}::%'",
|
|
855
|
+
)
|
|
856
|
+
return {obj.name.rpartition('::')[-1]: obj for obj in objects}
|
|
857
|
+
|
|
858
|
+
@future_property(on_demand=True)
|
|
859
|
+
async def system_objects(self) -> Dict[str, edgedb.Object]:
|
|
860
|
+
"""当前space下所有的系统对象"""
|
|
861
|
+
objects = await AsyncDeepModel.query_object(
|
|
862
|
+
self,
|
|
863
|
+
f"{OBJECT_QUERY} filter .name like '{self.spacemodule}::%'",
|
|
864
|
+
)
|
|
865
|
+
return {obj.name.rpartition('::')[-1]: obj for obj in objects}
|
|
866
|
+
|
|
867
|
+
@cached_property
|
|
868
|
+
def objects(self) -> Dict[str, ObjectTypeFrame]:
|
|
869
|
+
result = {name: format_obj(obj) for name, obj in self.user_objects.items()}
|
|
870
|
+
for name, obj in self.system_objects.items():
|
|
871
|
+
# In case of duplicate name both in app obj& space obj
|
|
872
|
+
if name in result:
|
|
873
|
+
result[f'{self.spacemodule}::{name}'] = format_obj(obj)
|
|
874
|
+
else:
|
|
875
|
+
result[name] = format_obj(obj)
|
|
876
|
+
|
|
877
|
+
return result
|
|
878
|
+
|
|
879
|
+
@staticmethod
|
|
880
|
+
def _prepare_variables(kwargs):
|
|
881
|
+
variables = {}
|
|
882
|
+
for k, v in kwargs.items():
|
|
883
|
+
variables[str(k)] = v
|
|
884
|
+
return variables
|
|
885
|
+
|
|
886
|
+
async def query_object(self, ql: str, **kwargs) -> List[Any]:
|
|
887
|
+
"""执行ql查询语句,得到原始结果返回
|
|
888
|
+
|
|
889
|
+
如有变量,以kwargs的方式提供
|
|
890
|
+
|
|
891
|
+
Args:
|
|
892
|
+
ql: 执行的ql
|
|
893
|
+
|
|
894
|
+
See Also:
|
|
895
|
+
|
|
896
|
+
:func:`query`, 执行ql查询语句,得到序列化后的结果
|
|
897
|
+
:func:`query_df`, 执行ql查询语句,获取DataFrame格式的二维表
|
|
898
|
+
|
|
899
|
+
"""
|
|
900
|
+
|
|
901
|
+
if self.direct_access:
|
|
902
|
+
logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
|
|
903
|
+
_, result = await self.client.query(ql, **kwargs)
|
|
904
|
+
return result
|
|
905
|
+
|
|
906
|
+
result = await AsyncDeepModel.query(self, ql, **kwargs)
|
|
907
|
+
return serutils.deserialize(result)
|
|
908
|
+
|
|
909
|
+
async def query(self, ql: str, **kwargs) -> List[Any]:
|
|
910
|
+
"""执行ql查询语句,得到序列化后的结果
|
|
911
|
+
|
|
912
|
+
如有变量,以args, kwargs的方式提供
|
|
913
|
+
|
|
914
|
+
Args:
|
|
915
|
+
ql: 执行的ql
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
.. admonition:: 示例
|
|
919
|
+
|
|
920
|
+
.. code-block:: python
|
|
921
|
+
|
|
922
|
+
dm = DeepModel()
|
|
923
|
+
|
|
924
|
+
# 以变量name 查询User对象
|
|
925
|
+
dm.query(
|
|
926
|
+
'select User{name, is_active} filter .name=<std::str>$name',
|
|
927
|
+
name='Alice'
|
|
928
|
+
)
|
|
929
|
+
|
|
930
|
+
See Also:
|
|
931
|
+
|
|
932
|
+
:func:`query_df`, 执行ql查询语句,获取DataFrame格式的二维表
|
|
933
|
+
:func:`query_object`, 执行ql查询语句,得到原始结果返回
|
|
934
|
+
|
|
935
|
+
"""
|
|
936
|
+
if self.direct_access:
|
|
937
|
+
logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
|
|
938
|
+
frame_desc, result = await self.client.query(ql, **kwargs)
|
|
939
|
+
return serutils.serialize(
|
|
940
|
+
result, ctx=serutils.Context(frame_desc=frame_desc)
|
|
941
|
+
)
|
|
942
|
+
|
|
943
|
+
result = await self._http_query(ql, **kwargs)
|
|
944
|
+
return result.json_
|
|
945
|
+
|
|
946
|
+
async def _http_query(self, ql: str, **kwargs) -> QueryResult:
|
|
947
|
+
logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
|
|
948
|
+
result = await self.async_api.deepql.query(
|
|
949
|
+
module=self.appmodule,
|
|
950
|
+
query=ql,
|
|
951
|
+
variables=self._prepare_variables(kwargs)
|
|
952
|
+
)
|
|
953
|
+
self._maybe_handle_error(result.json_)
|
|
954
|
+
return result
|
|
955
|
+
|
|
956
|
+
async def query_df(self, ql: str, **kwargs) -> pd.DataFrame:
|
|
957
|
+
"""执行ql查询语句
|
|
958
|
+
|
|
959
|
+
获取DataFrame格式的二维表
|
|
960
|
+
如有变量,以kwargs的方式提供
|
|
961
|
+
|
|
962
|
+
Args:
|
|
963
|
+
ql: 执行的ql
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
.. admonition:: 示例
|
|
967
|
+
|
|
968
|
+
.. code-block:: python
|
|
969
|
+
|
|
970
|
+
dm = DeepModel()
|
|
971
|
+
|
|
972
|
+
# 以变量name 查询User对象,得到DataFrame
|
|
973
|
+
dm.query_df(
|
|
974
|
+
'select User{name, is_active} filter .name=<std::str>$name',
|
|
975
|
+
name='Alice'
|
|
976
|
+
)
|
|
977
|
+
|
|
978
|
+
See Also:
|
|
979
|
+
|
|
980
|
+
:func:`query`, 执行ql查询语句,得到序列化后的结果
|
|
981
|
+
:func:`query_object`, 执行ql查询语句,得到原始结果返回
|
|
982
|
+
|
|
983
|
+
"""
|
|
984
|
+
if self.direct_access:
|
|
985
|
+
frame_desc, data = await self.client.query(ql, **kwargs)
|
|
986
|
+
|
|
987
|
+
data = pd.DataFrame(data=serutils.serialize(
|
|
988
|
+
data, ctx=serutils.Context(frame_desc=frame_desc, query_df=True)
|
|
989
|
+
))
|
|
990
|
+
# Not records for dict-like
|
|
991
|
+
if not isinstance(frame_desc, dict):
|
|
992
|
+
return data
|
|
993
|
+
|
|
994
|
+
structure = collect_frame_desc_structure(frame_desc)
|
|
995
|
+
else:
|
|
996
|
+
result = await self._http_query(ql, **kwargs)
|
|
997
|
+
# No object structure info
|
|
998
|
+
if result.objectInfos is None:
|
|
999
|
+
return pd.DataFrame(data=result.json_)
|
|
1000
|
+
|
|
1001
|
+
data = pd.DataFrame(
|
|
1002
|
+
data=result.json_,
|
|
1003
|
+
columns=[f.name for f in result.objectInfos[0].fields]
|
|
1004
|
+
)
|
|
1005
|
+
|
|
1006
|
+
structure = collect_query_result_structure(result.objectInfos[0])
|
|
1007
|
+
|
|
1008
|
+
if data.empty:
|
|
1009
|
+
return pd.DataFrame(columns=structure.fields.keys())
|
|
1010
|
+
|
|
1011
|
+
structure.cast(data, self.direct_access)
|
|
1012
|
+
return data
|
|
1013
|
+
|
|
1014
|
+
query.__doc__ = query.__doc__ + DOC_ARGS_KWARGS
|
|
1015
|
+
query_object.__doc__ = query_object.__doc__ + DOC_ARGS_KWARGS
|
|
1016
|
+
query_df.__doc__ = query_df.__doc__ + DOC_ARGS_KWARGS
|
|
1017
|
+
|
|
1018
|
+
@txn_support
|
|
1019
|
+
async def execute(
|
|
1020
|
+
self,
|
|
1021
|
+
qls: Union[str, List[str], List[QueryWithArgs]],
|
|
1022
|
+
**kwargs
|
|
1023
|
+
) -> Optional[List]:
|
|
1024
|
+
"""以事务执行多句ql
|
|
1025
|
+
|
|
1026
|
+
Args:
|
|
1027
|
+
qls: 要执行的若干ql语句
|
|
1028
|
+
可通过提供QueryWithArgs对象ql的方式定制每句ql的参数信息
|
|
1029
|
+
亦可直接以kwargs的形式提供参数信息
|
|
1030
|
+
会自动用作所有string形式ql的参数
|
|
1031
|
+
|
|
1032
|
+
"""
|
|
1033
|
+
if isinstance(qls, str):
|
|
1034
|
+
qls_with_args = [QueryWithArgs(commands=qls, kwargs=kwargs)]
|
|
1035
|
+
else:
|
|
1036
|
+
qls_with_args = []
|
|
1037
|
+
seen_kwargs_key = set()
|
|
1038
|
+
for ql in qls:
|
|
1039
|
+
if isinstance(ql, QueryWithArgs):
|
|
1040
|
+
if (
|
|
1041
|
+
not self.direct_access
|
|
1042
|
+
and ql.kwargs
|
|
1043
|
+
and seen_kwargs_key.intersection(ql.kwargs.keys())
|
|
1044
|
+
):
|
|
1045
|
+
raise NotImplementedError('非直连模式不支持重名variables')
|
|
1046
|
+
|
|
1047
|
+
qls_with_args.append(ql)
|
|
1048
|
+
if ql.kwargs:
|
|
1049
|
+
seen_kwargs_key = seen_kwargs_key.union(ql.kwargs.keys())
|
|
1050
|
+
|
|
1051
|
+
elif isinstance(ql, str):
|
|
1052
|
+
qls_with_args.append(QueryWithArgs(commands=ql, kwargs=kwargs))
|
|
1053
|
+
else:
|
|
1054
|
+
raise TypeError(f'qls参数中出现类型非法成员:{type(ql)}')
|
|
1055
|
+
|
|
1056
|
+
return await self._maybe_exec_qls(qls_with_args)
|
|
1057
|
+
|
|
1058
|
+
execute.__doc__ = execute.__doc__ + DOC_ARGS_KWARGS
|
|
1059
|
+
|
|
1060
|
+
async def _execute(self, qls_with_args: List[QueryWithArgs]) -> List:
|
|
1061
|
+
self.alias.reset(BATCH_INSERT_KW)
|
|
1062
|
+
if not self.direct_access:
|
|
1063
|
+
kwargs = {}
|
|
1064
|
+
seen_kwargs_key = set()
|
|
1065
|
+
|
|
1066
|
+
for ql in qls_with_args:
|
|
1067
|
+
if ql.kwargs and seen_kwargs_key.intersection(ql.kwargs.keys()):
|
|
1068
|
+
raise NotImplementedError('非直连模式不支持重名variables')
|
|
1069
|
+
if ql.kwargs:
|
|
1070
|
+
kwargs.update(ql.kwargs)
|
|
1071
|
+
seen_kwargs_key = seen_kwargs_key.union(ql.kwargs.keys())
|
|
1072
|
+
|
|
1073
|
+
commands = ';'.join([ql.commands for ql in qls_with_args])
|
|
1074
|
+
|
|
1075
|
+
logger.opt(lazy=True).debug(
|
|
1076
|
+
f"Execute QL: [{commands}], \nkwargs: [{kwargs}]."
|
|
1077
|
+
)
|
|
1078
|
+
res = await self.async_api.deepql.execute(
|
|
1079
|
+
module=self.appmodule,
|
|
1080
|
+
query=commands,
|
|
1081
|
+
variables=self._prepare_variables(kwargs)
|
|
1082
|
+
)
|
|
1083
|
+
affected = res.get('json')
|
|
1084
|
+
self._maybe_handle_error(affected)
|
|
1085
|
+
return affected
|
|
1086
|
+
|
|
1087
|
+
result = []
|
|
1088
|
+
async for tx in self.client.transaction():
|
|
1089
|
+
async with tx:
|
|
1090
|
+
for ql in qls_with_args:
|
|
1091
|
+
logger.opt(lazy=True).debug(
|
|
1092
|
+
f"Execute QL: [{ql.commands}], \nkwargs: [{ql.kwargs}]."
|
|
1093
|
+
)
|
|
1094
|
+
desc, affected = await tx.execute(ql.commands, **ql.kwargs)
|
|
1095
|
+
result.append(serutils.serialize(
|
|
1096
|
+
affected, ctx=serutils.Context(frame_desc=desc)
|
|
1097
|
+
))
|
|
1098
|
+
if len(result) == 1:
|
|
1099
|
+
return result[0]
|
|
1100
|
+
return result
|
|
1101
|
+
|
|
1102
|
+
@staticmethod
|
|
1103
|
+
def _maybe_handle_error(res):
|
|
1104
|
+
if not isinstance(res, dict):
|
|
1105
|
+
return
|
|
1106
|
+
|
|
1107
|
+
if error := res.get('errors'): # pragma: no cover
|
|
1108
|
+
ex_msg = error['message'].strip()
|
|
1109
|
+
ex_code = int(error['code'])
|
|
1110
|
+
raise edgedb.EdgeDBError._from_code(ex_code, ex_msg) # noqa
|
|
1111
|
+
|
|
1112
|
+
async def _maybe_exec_qls(
|
|
1113
|
+
self,
|
|
1114
|
+
qls_with_args: List[QueryWithArgs]
|
|
1115
|
+
) -> Optional[List]:
|
|
1116
|
+
txn_conf = self._safe_get_txn_conf()
|
|
1117
|
+
|
|
1118
|
+
if txn_conf.in_txn[-1] and self._txn_support_:
|
|
1119
|
+
txn_conf.qls[-1].extend(qls_with_args)
|
|
1120
|
+
return
|
|
1121
|
+
|
|
1122
|
+
return await self._execute(qls_with_args)
|
|
1123
|
+
|
|
1124
|
+
@staticmethod
|
|
1125
|
+
def _valid_data(data, object_name, relation, structure):
|
|
1126
|
+
required_fields = set(map(
|
|
1127
|
+
lambda f: f.name,
|
|
1128
|
+
filter(lambda f: f.required, structure.fields.values())
|
|
1129
|
+
))
|
|
1130
|
+
if missing_fields := (required_fields - set(data.columns)):
|
|
1131
|
+
raise RequiredFieldUnfilled(f'缺少必填字段: {missing_fields}')
|
|
1132
|
+
|
|
1133
|
+
if not relation:
|
|
1134
|
+
return
|
|
1135
|
+
|
|
1136
|
+
for name, link_df in relation.items():
|
|
1137
|
+
if name not in structure.fields:
|
|
1138
|
+
continue
|
|
1139
|
+
field = structure.fields[name]
|
|
1140
|
+
if not field.is_multi_link:
|
|
1141
|
+
if name in data.columns:
|
|
1142
|
+
continue
|
|
1143
|
+
|
|
1144
|
+
raise SingleLinkInRelation(
|
|
1145
|
+
f'对象[{object_name}]的Link:[{name}]非multi link, '
|
|
1146
|
+
f'请直接作为入参data的{name}列提供, '
|
|
1147
|
+
f'值为对象{structure.fields[name].type}的业务主键'
|
|
1148
|
+
)
|
|
1149
|
+
|
|
1150
|
+
if not {'source', 'target'}.issubset(link_df.columns):
|
|
1151
|
+
raise ValueError("关联表必须包含source和target列")
|
|
1152
|
+
|
|
1153
|
+
@staticmethod
|
|
1154
|
+
def _valid_upsert(
|
|
1155
|
+
obj: ObjectTypeFrame,
|
|
1156
|
+
field_names: Iterable[str],
|
|
1157
|
+
bkey: str,
|
|
1158
|
+
exclusive_fields: List[str] = None,
|
|
1159
|
+
update_fields: List[str] = None
|
|
1160
|
+
):
|
|
1161
|
+
if (
|
|
1162
|
+
update_fields
|
|
1163
|
+
and (missing := (set(update_fields) - set(field_names)))
|
|
1164
|
+
):
|
|
1165
|
+
raise ValueError(f"update fields: {missing} 不在提供的数据中")
|
|
1166
|
+
|
|
1167
|
+
if exclusive_fields:
|
|
1168
|
+
if missing := (set(exclusive_fields) - set(field_names)):
|
|
1169
|
+
raise ValueError(f"exclusive fields: {missing} 不在提供的数据中")
|
|
1170
|
+
|
|
1171
|
+
valid_exclusive = [*obj.exclusive_fields, bkey]
|
|
1172
|
+
exclusive = (
|
|
1173
|
+
exclusive_fields[0]
|
|
1174
|
+
if len(exclusive_fields) == 1 else set(exclusive_fields)
|
|
1175
|
+
)
|
|
1176
|
+
if exclusive not in valid_exclusive:
|
|
1177
|
+
raise ValueError(f"exclusive fields: {exclusive_fields} 没有相应唯一约束")
|
|
1178
|
+
|
|
1179
|
+
@txn_support
|
|
1180
|
+
async def insert_df(
|
|
1181
|
+
self,
|
|
1182
|
+
object_name: str,
|
|
1183
|
+
data: pd.DataFrame,
|
|
1184
|
+
relation: Dict[str, pd.DataFrame] = None,
|
|
1185
|
+
chunksize: int = 500,
|
|
1186
|
+
enable_upsert: bool = False,
|
|
1187
|
+
update_fields: Iterable[str] = None,
|
|
1188
|
+
exclusive_fields: Iterable[str] = None,
|
|
1189
|
+
) -> None:
|
|
1190
|
+
"""以事务执行基于DataFrame字段信息的批量插入数据
|
|
1191
|
+
|
|
1192
|
+
Args:
|
|
1193
|
+
object_name: 被插入数据的对象名,需属于当前应用
|
|
1194
|
+
data: 要插入的数据,若有single link property,
|
|
1195
|
+
则以列名为link_name@link_property_name的形式提供
|
|
1196
|
+
relation: 如有multi link,提供该字典用于补充link target信息,
|
|
1197
|
+
键为link字段名,值为映射关系的DataFrame
|
|
1198
|
+
DataFrame中的source列需为插入对象的业务主键,
|
|
1199
|
+
target列需为link target的业务主键,
|
|
1200
|
+
若有link property,则以property名为列名,提供在除source和target的列中
|
|
1201
|
+
chunksize: 单次最大行数
|
|
1202
|
+
enable_upsert: 是否组织成upsert句式
|
|
1203
|
+
update_fields: upsert句式下update的update fields列表,
|
|
1204
|
+
涉及的fields需出现在data或relation中,
|
|
1205
|
+
默认为提供的data列中除业务主键以外的fields
|
|
1206
|
+
exclusive_fields: upsert句式下update的exclusive fields列表,
|
|
1207
|
+
涉及的fields需出现在data或relation中,
|
|
1208
|
+
默认为业务主键
|
|
1209
|
+
|
|
1210
|
+
Notes:
|
|
1211
|
+
|
|
1212
|
+
由于批量insert实现方式为组织 for-union clause 的 insert 语句,
|
|
1213
|
+
而在其中查询self link只能查到已有数据,
|
|
1214
|
+
无法查到 for-union clause 之前循环插入的结果,self link字段的数据将被单独抽出,
|
|
1215
|
+
在 insert 后再用 for-union clause 的 update 语句更新
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
.. admonition:: 示例(不涉及multi link)
|
|
1219
|
+
|
|
1220
|
+
.. code-block:: python
|
|
1221
|
+
|
|
1222
|
+
import pandas as pd
|
|
1223
|
+
from datetime import datetime
|
|
1224
|
+
|
|
1225
|
+
dm = DeepModel()
|
|
1226
|
+
|
|
1227
|
+
data = pd.DataFrame(
|
|
1228
|
+
{
|
|
1229
|
+
'p_bool': [True, False],
|
|
1230
|
+
'p_str': ['Hello', 'World'],
|
|
1231
|
+
'p_local_datetime': [
|
|
1232
|
+
datetime(2021, 1, 1, 0, 0, 0),
|
|
1233
|
+
datetime(2021, 2, 1, 0, 0, 0),
|
|
1234
|
+
],
|
|
1235
|
+
}
|
|
1236
|
+
)
|
|
1237
|
+
# 将data插入Demo对象
|
|
1238
|
+
dm.insert_df('Demo', data)
|
|
1239
|
+
|
|
1240
|
+
.. admonition:: 示例(涉及multi link)
|
|
1241
|
+
|
|
1242
|
+
.. code-block:: python
|
|
1243
|
+
|
|
1244
|
+
import pandas as pd
|
|
1245
|
+
|
|
1246
|
+
dm = DeepModel()
|
|
1247
|
+
# 对象主数据
|
|
1248
|
+
data = pd.DataFrame(
|
|
1249
|
+
{
|
|
1250
|
+
'name': ['Alice', 'Bob', 'Carol']
|
|
1251
|
+
}
|
|
1252
|
+
)
|
|
1253
|
+
# 主数据的multi link字段target信息
|
|
1254
|
+
relation = {
|
|
1255
|
+
'deck': pd.DataFrame(
|
|
1256
|
+
{
|
|
1257
|
+
# 一对多可用多行source与target的关联表示
|
|
1258
|
+
'source': ['Alice', 'Alice', 'Bob', 'Carol'],
|
|
1259
|
+
'target': [
|
|
1260
|
+
'Dragon',
|
|
1261
|
+
'Golem',
|
|
1262
|
+
'Golem',
|
|
1263
|
+
'Imp'
|
|
1264
|
+
]
|
|
1265
|
+
}
|
|
1266
|
+
),
|
|
1267
|
+
'awards': pd.DataFrame(
|
|
1268
|
+
{
|
|
1269
|
+
'source': ['Alice', 'Bob', 'Carol'],
|
|
1270
|
+
'target': ['1st', '2nd', '3rd'],
|
|
1271
|
+
}
|
|
1272
|
+
)
|
|
1273
|
+
}
|
|
1274
|
+
dm.insert_df('User', data, relation=relation)
|
|
1275
|
+
|
|
1276
|
+
"""
|
|
1277
|
+
if data.empty:
|
|
1278
|
+
logger.info("data为空,无DML执行")
|
|
1279
|
+
return
|
|
1280
|
+
|
|
1281
|
+
if object_name in self.objects:
|
|
1282
|
+
obj = self.objects[object_name]
|
|
1283
|
+
else:
|
|
1284
|
+
raise ObjectNotExist(
|
|
1285
|
+
f'DeepModel对象[{object_name}]在当前应用不存在,无法插入数据'
|
|
1286
|
+
)
|
|
1287
|
+
if obj.external:
|
|
1288
|
+
raise ExternalObjectReadOnly('外部对象只可读')
|
|
1289
|
+
|
|
1290
|
+
structure = ObjectStructure(name=obj.name, structure=obj.fields.values())
|
|
1291
|
+
|
|
1292
|
+
self._valid_data(data, object_name, relation, structure)
|
|
1293
|
+
|
|
1294
|
+
relation = relation or {}
|
|
1295
|
+
bkey = await self._get_bkey(obj)
|
|
1296
|
+
# data拼接relation df
|
|
1297
|
+
data = self._merge_relation(data, relation, structure, bkey)
|
|
1298
|
+
# 从data中分离出self-link更新信息
|
|
1299
|
+
data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
|
|
1300
|
+
|
|
1301
|
+
field_info = []
|
|
1302
|
+
tgt_main_field = {}
|
|
1303
|
+
# 准备bulk insert所需field信息
|
|
1304
|
+
for field in structure.fields.values():
|
|
1305
|
+
if field.name not in data.columns:
|
|
1306
|
+
continue
|
|
1307
|
+
|
|
1308
|
+
field_info.append(field)
|
|
1309
|
+
|
|
1310
|
+
if not field.is_link:
|
|
1311
|
+
continue
|
|
1312
|
+
|
|
1313
|
+
is_multi = field.is_multi_link
|
|
1314
|
+
name = field.name
|
|
1315
|
+
# 链接至其他对象,记录目标对象信息
|
|
1316
|
+
if is_multi:
|
|
1317
|
+
link_props = set(relation[name].columns).intersection(field.props)
|
|
1318
|
+
else:
|
|
1319
|
+
link_props = set(
|
|
1320
|
+
c[len(f'{name}@')::]
|
|
1321
|
+
for c in data.columns if c.startswith(f'{name}@')
|
|
1322
|
+
).intersection(field.props)
|
|
1323
|
+
tgt_bkey = await self._get_bkey(field.target, object_name, name)
|
|
1324
|
+
tgt_main_field[name] = MainField(tgt_bkey, is_multi, link_props)
|
|
1325
|
+
|
|
1326
|
+
field_names = set(map(lambda f: f.name, field_info))
|
|
1327
|
+
if enable_upsert:
|
|
1328
|
+
self._valid_upsert(obj, field_names, bkey, exclusive_fields, update_fields)
|
|
1329
|
+
|
|
1330
|
+
exclusive_fields = set(exclusive_fields or {bkey}) & set(field_names)
|
|
1331
|
+
update_fields = set(update_fields or (field_names- {bkey})) & set(field_names)
|
|
1332
|
+
if enable_upsert and update_fields:
|
|
1333
|
+
insert_ql = bulk_upsert_by_fields(
|
|
1334
|
+
object_name, field_info, tgt_main_field,
|
|
1335
|
+
exclusive_fields, update_fields
|
|
1336
|
+
)
|
|
1337
|
+
else:
|
|
1338
|
+
insert_ql = bulk_insert_by_fields(object_name, field_info, tgt_main_field)
|
|
1339
|
+
|
|
1340
|
+
qls = []
|
|
1341
|
+
self._collect_qls(structure.fit(data), insert_ql, chunksize, qls)
|
|
1342
|
+
if self_link_dfs:
|
|
1343
|
+
for update_field, (update_df, main_field) in self_link_dfs.items():
|
|
1344
|
+
field = structure.fields[update_field]
|
|
1345
|
+
update_ql = bulk_update_by_fields(
|
|
1346
|
+
object_name, bkey, [field],
|
|
1347
|
+
{update_field: main_field}, [update_field]
|
|
1348
|
+
)
|
|
1349
|
+
self._collect_qls(update_df, update_ql, chunksize, qls)
|
|
1350
|
+
|
|
1351
|
+
await self.execute(qls)
|
|
1352
|
+
|
|
1353
|
+
def _collect_qls(
|
|
1354
|
+
self,
|
|
1355
|
+
data: pd.DataFrame,
|
|
1356
|
+
ql: str,
|
|
1357
|
+
chunksize: int,
|
|
1358
|
+
qls: List[QueryWithArgs]
|
|
1359
|
+
):
|
|
1360
|
+
for i in range(0, len(data), chunksize):
|
|
1361
|
+
part = data.iloc[i: i + chunksize]
|
|
1362
|
+
kw_name = self.alias.get(BATCH_INSERT_KW)
|
|
1363
|
+
qls.append(QueryWithArgs(
|
|
1364
|
+
commands=ql.replace(
|
|
1365
|
+
f'${BATCH_INSERT_KW}', f'${kw_name}'
|
|
1366
|
+
),
|
|
1367
|
+
kwargs={kw_name: part.to_json(orient='records', double_precision=15)}
|
|
1368
|
+
))
|
|
1369
|
+
|
|
1370
|
+
async def get_object(
|
|
1371
|
+
self,
|
|
1372
|
+
object_name: str,
|
|
1373
|
+
raw: bool = True
|
|
1374
|
+
) -> Optional[Union[ObjectTypeFrame, edgedb.Object]]:
|
|
1375
|
+
"""获取指定对象名的对象
|
|
1376
|
+
|
|
1377
|
+
Args:
|
|
1378
|
+
object_name: 对象名,如对象名未提供module name部分,则默认module name为当前app module
|
|
1379
|
+
如需查询其他module的对象,应以 module_name::object_name 的形式提供
|
|
1380
|
+
raw: 是否获取edgedb.Object数据,为False时将转换为ObjectTypeFrame数据
|
|
1381
|
+
默认为True
|
|
1382
|
+
|
|
1383
|
+
"""
|
|
1384
|
+
if '::' not in object_name:
|
|
1385
|
+
object_name = f'{self.appmodule}::{object_name}'
|
|
1386
|
+
objs = await AsyncDeepModel.query_object(
|
|
1387
|
+
self,
|
|
1388
|
+
f"{OBJECT_QUERY} filter .name = '{object_name}'",
|
|
1389
|
+
)
|
|
1390
|
+
if len(objs) == 0:
|
|
1391
|
+
return
|
|
1392
|
+
|
|
1393
|
+
if raw:
|
|
1394
|
+
return objs[0]
|
|
1395
|
+
|
|
1396
|
+
return format_obj(objs[0])
|
|
1397
|
+
|
|
1398
|
+
async def _get_bkey(
|
|
1399
|
+
self,
|
|
1400
|
+
obj: Union[ObjectTypeFrame, TargetField],
|
|
1401
|
+
source: str = None,
|
|
1402
|
+
name: str = None
|
|
1403
|
+
) -> str:
|
|
1404
|
+
# 如可在object结构的annotations中取业务主键,则优先取,否则走接口
|
|
1405
|
+
if obj.info and BUSINESS_KEY in obj.info:
|
|
1406
|
+
return obj.info[BUSINESS_KEY]
|
|
1407
|
+
elif (code := obj.normalized_name) in self.model_objects:
|
|
1408
|
+
return self.model_objects[code].businessKey
|
|
1409
|
+
|
|
1410
|
+
assert isinstance(obj, TargetField)
|
|
1411
|
+
# Link 至非本应用对象,需单独查询
|
|
1412
|
+
tgt = ObjectElement.construct_from(self.model_objects[source]).links[name]
|
|
1413
|
+
tgt_model_info = await self.async_api.object.info(
|
|
1414
|
+
app=tgt.targetApp, object_code=tgt.targetObjectCode
|
|
1415
|
+
)
|
|
1416
|
+
return tgt_model_info.businessKey
|
|
1417
|
+
|
|
1418
|
+
@staticmethod
|
|
1419
|
+
def _split_self_link(data, relation, structure, bkey):
|
|
1420
|
+
self_link_dfs = {}
|
|
1421
|
+
for name in structure.self_link_fields:
|
|
1422
|
+
field = structure.fields[name]
|
|
1423
|
+
if (link_df := relation.get(name)) is not None:
|
|
1424
|
+
link_props = set(link_df.columns).intersection(field.props)
|
|
1425
|
+
self_link_dfs[name] = (
|
|
1426
|
+
structure.fit(data[[bkey, name]]),
|
|
1427
|
+
MainField(bkey, field.is_multi_link, link_props)
|
|
1428
|
+
)
|
|
1429
|
+
data = data.drop(columns=[name])
|
|
1430
|
+
elif name in data.columns:
|
|
1431
|
+
link_prop_cols = []
|
|
1432
|
+
link_props = []
|
|
1433
|
+
|
|
1434
|
+
for col in data.columns:
|
|
1435
|
+
if (
|
|
1436
|
+
col.startswith(f'{name}@')
|
|
1437
|
+
and ((prop_name := col[len(f'{name}@')::]) in field.props)
|
|
1438
|
+
):
|
|
1439
|
+
link_prop_cols.append(col)
|
|
1440
|
+
link_props.append(prop_name)
|
|
1441
|
+
|
|
1442
|
+
self_link_dfs[name] = (
|
|
1443
|
+
structure.fit(data[[bkey, name, *link_prop_cols]]),
|
|
1444
|
+
MainField(bkey, field.is_multi_link, link_props)
|
|
1445
|
+
)
|
|
1446
|
+
data = data.drop(columns=[name, *link_prop_cols])
|
|
1447
|
+
return data, self_link_dfs
|
|
1448
|
+
|
|
1449
|
+
@staticmethod
|
|
1450
|
+
def _merge_relation(data, relation, structure, bkey):
|
|
1451
|
+
for name, link_df in relation.items():
|
|
1452
|
+
if name not in structure.fields:
|
|
1453
|
+
continue
|
|
1454
|
+
field = structure.fields[name]
|
|
1455
|
+
valid_cols = list({'source', 'target', *field.props} & set(link_df.columns))
|
|
1456
|
+
link_df = link_df[valid_cols]
|
|
1457
|
+
# for fit only
|
|
1458
|
+
temp_structure = ObjectStructure(
|
|
1459
|
+
field.type,
|
|
1460
|
+
[
|
|
1461
|
+
PtrInfo(name='source', target=TargetField(name='std::str')),
|
|
1462
|
+
PtrInfo(name='target', target=TargetField(name='std::str')),
|
|
1463
|
+
*[PtrInfo(**prop.dict()) for prop in field.properties]
|
|
1464
|
+
]
|
|
1465
|
+
)
|
|
1466
|
+
link_df = temp_structure.fit(link_df)
|
|
1467
|
+
link = link_df.groupby('source').apply(_format_link, link_name=name)
|
|
1468
|
+
data = data.drop(columns=[name], errors='ignore')
|
|
1469
|
+
data = data.join(link.to_frame(name), on=bkey)
|
|
1470
|
+
return data
|
|
1471
|
+
|
|
1472
|
+
async def insert_df_pg(
|
|
1473
|
+
self,
|
|
1474
|
+
object_name: str,
|
|
1475
|
+
data: pd.DataFrame,
|
|
1476
|
+
chunk_size: int = 500,
|
|
1477
|
+
enable_upsert: bool = False,
|
|
1478
|
+
exclusive_fields: Iterable[str] = None,
|
|
1479
|
+
update_fields: Iterable[str] = None,
|
|
1480
|
+
) -> None:
|
|
1481
|
+
"""以事务直连pg执行基于DataFrame字段信息的批量插入数据
|
|
1482
|
+
|
|
1483
|
+
Args:
|
|
1484
|
+
object_name: 被插入数据的对象名,需属于当前应用
|
|
1485
|
+
data: 要插入的数据,只支持single property/link,且single link不可包含link property
|
|
1486
|
+
chunk_size: 单次最大行数
|
|
1487
|
+
enable_upsert: 是否组织成insert-on conflict do update 句式
|
|
1488
|
+
exclusive_fields: enable_upsert为True时的conflict字段,默认为business key
|
|
1489
|
+
update_fields: enable_upsert为True时的update字段,默认为除business key外所有已提供的字段
|
|
1490
|
+
|
|
1491
|
+
"""
|
|
1492
|
+
if self.pg_dsn is None:
|
|
1493
|
+
raise ValueError('pg_dsn未提供')
|
|
1494
|
+
|
|
1495
|
+
if data.empty:
|
|
1496
|
+
return
|
|
1497
|
+
|
|
1498
|
+
if object_name in self.objects:
|
|
1499
|
+
obj = self.objects[object_name]
|
|
1500
|
+
else:
|
|
1501
|
+
raise ObjectNotExist(
|
|
1502
|
+
f'DeepModel对象[{object_name}]在当前应用不存在,无法插入数据'
|
|
1503
|
+
)
|
|
1504
|
+
if obj.external:
|
|
1505
|
+
raise ExternalObjectReadOnly('外部对象只可读')
|
|
1506
|
+
|
|
1507
|
+
structure = ObjectStructure(
|
|
1508
|
+
name=obj.name,
|
|
1509
|
+
structure=obj.fields.values(),
|
|
1510
|
+
include_id=True
|
|
1511
|
+
)
|
|
1512
|
+
self._valid_data(data, object_name, {}, structure)
|
|
1513
|
+
|
|
1514
|
+
field_info = []
|
|
1515
|
+
tgt_info = {}
|
|
1516
|
+
field_map = structure.fields
|
|
1517
|
+
bkey = await self._get_bkey(obj)
|
|
1518
|
+
|
|
1519
|
+
data = data.drop(columns=['id'], errors='ignore')
|
|
1520
|
+
data = data.assign(id=[uuid.uuid4() for _ in data.index], index=data.index)
|
|
1521
|
+
|
|
1522
|
+
for name, field in field_map.items():
|
|
1523
|
+
if name not in data.columns:
|
|
1524
|
+
continue
|
|
1525
|
+
if '@' in name:
|
|
1526
|
+
raise NotImplementedError('不支持插入包含link property的数据')
|
|
1527
|
+
if field.props:
|
|
1528
|
+
raise NotImplementedError('不支持插入有link property的link数据')
|
|
1529
|
+
|
|
1530
|
+
field_info.append(field)
|
|
1531
|
+
if not field.is_link:
|
|
1532
|
+
continue
|
|
1533
|
+
|
|
1534
|
+
if field.is_multi:
|
|
1535
|
+
raise NotImplementedError('不支持插入包含multi link/property的数据')
|
|
1536
|
+
|
|
1537
|
+
tgt = field.target
|
|
1538
|
+
tgt_bkey = await self._get_bkey(field.target, object_name, name)
|
|
1539
|
+
tgt_bkey_col = str(tgt.props[tgt_bkey].id)
|
|
1540
|
+
if tgt_col := tgt.props.get(field.target_col):
|
|
1541
|
+
tgt_info[name] = MainPgCol(
|
|
1542
|
+
target_bkey_col=tgt_bkey_col,
|
|
1543
|
+
target_col=str(tgt_col.id),
|
|
1544
|
+
)
|
|
1545
|
+
else:
|
|
1546
|
+
tgt_info[name] = MainPgCol(target_bkey_col=tgt_bkey_col, target_col='id')
|
|
1547
|
+
|
|
1548
|
+
# field_names will be used for data col order
|
|
1549
|
+
# so here use field_set as the set for field_names
|
|
1550
|
+
field_names = list(map(lambda f: f.name, field_info))
|
|
1551
|
+
field_set = set(field_names)
|
|
1552
|
+
|
|
1553
|
+
if enable_upsert:
|
|
1554
|
+
self._valid_upsert(obj, field_set, bkey, exclusive_fields, update_fields)
|
|
1555
|
+
|
|
1556
|
+
exclusive_fields = set(exclusive_fields or {bkey}) & field_set
|
|
1557
|
+
update_fields = (
|
|
1558
|
+
set(update_fields or (field_set - {bkey})) & field_set
|
|
1559
|
+
) - {'id'}
|
|
1560
|
+
|
|
1561
|
+
update_fields = list(map(lambda uf: field_map[uf], update_fields))
|
|
1562
|
+
exclusive_fields = list(map(lambda ex: field_map[ex], exclusive_fields))
|
|
1563
|
+
|
|
1564
|
+
if enable_upsert:
|
|
1565
|
+
main_sql = batch_insert_pg(
|
|
1566
|
+
str(obj.id), field_info, tgt_info,
|
|
1567
|
+
exclusive_fields, update_fields
|
|
1568
|
+
)
|
|
1569
|
+
else:
|
|
1570
|
+
main_sql = batch_insert_pg(str(obj.id), field_info, tgt_info)
|
|
1571
|
+
|
|
1572
|
+
data = structure.fit(data, raw_pg=True)
|
|
1573
|
+
data = data.fillna(np.nan)
|
|
1574
|
+
data = data.replace([np.nan], [None])
|
|
1575
|
+
conn = await pg_conn(dsn=self.pg_dsn)
|
|
1576
|
+
try:
|
|
1577
|
+
async with conn.transaction():
|
|
1578
|
+
logger.debug(f"Prepare SQL: [{main_sql}]")
|
|
1579
|
+
main_sql_st = await conn.prepare(main_sql)
|
|
1580
|
+
data = data[field_names]
|
|
1581
|
+
for i in range(0, len(data), chunk_size):
|
|
1582
|
+
part = data.iloc[i: i + chunk_size]
|
|
1583
|
+
await main_sql_st.executemany(part.itertuples(index=False, name=None))
|
|
1584
|
+
finally:
|
|
1585
|
+
await conn.close()
|
|
1586
|
+
|
|
1587
|
+
@asynccontextmanager
|
|
1588
|
+
async def start_transaction(self, flatten: bool = False):
|
|
1589
|
+
"""开启事务
|
|
1590
|
+
|
|
1591
|
+
上下文管理器,使用with语法开启上下文,上下文中的ql将作为事务执行
|
|
1592
|
+
退出with语句块后,事务将立即执行,执行过程中如果报错会直接抛出
|
|
1593
|
+
|
|
1594
|
+
.. admonition:: 示例
|
|
1595
|
+
|
|
1596
|
+
.. code-block:: python
|
|
1597
|
+
|
|
1598
|
+
import pandas as pd
|
|
1599
|
+
|
|
1600
|
+
dm = DeepModel()
|
|
1601
|
+
|
|
1602
|
+
data = pd.DataFrame(
|
|
1603
|
+
{
|
|
1604
|
+
'name': ['Alice', 'Bob', 'Carol'],
|
|
1605
|
+
'deck': [
|
|
1606
|
+
"Dragon",
|
|
1607
|
+
"Golem",
|
|
1608
|
+
"Imp"
|
|
1609
|
+
],
|
|
1610
|
+
'awards': [
|
|
1611
|
+
"1st",
|
|
1612
|
+
"2nd",
|
|
1613
|
+
"3rd"
|
|
1614
|
+
],
|
|
1615
|
+
}
|
|
1616
|
+
)
|
|
1617
|
+
|
|
1618
|
+
async with dm.start_transaction():
|
|
1619
|
+
await dm.execute("delete User")
|
|
1620
|
+
await dm.insert_df("User", data)
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
Important:
|
|
1624
|
+
|
|
1625
|
+
仅 :func:`insert_df` :func:`execute` 方法支持在事务中执行
|
|
1626
|
+
|
|
1627
|
+
"""
|
|
1628
|
+
try:
|
|
1629
|
+
self._txn_.get()
|
|
1630
|
+
except LookupError:
|
|
1631
|
+
self._txn_.set(_TxnConfig())
|
|
1632
|
+
|
|
1633
|
+
bak_flatten = self._txn_.get().flatten
|
|
1634
|
+
self._txn_.get().in_txn.append(True)
|
|
1635
|
+
|
|
1636
|
+
if flatten and not self._txn_.get().flatten:
|
|
1637
|
+
force_submit = True
|
|
1638
|
+
else:
|
|
1639
|
+
force_submit = False
|
|
1640
|
+
|
|
1641
|
+
self._txn_.get().flatten = bak_flatten or flatten
|
|
1642
|
+
|
|
1643
|
+
if not self._txn_.get().flatten:
|
|
1644
|
+
self._txn_.get().qls.append([])
|
|
1645
|
+
|
|
1646
|
+
try:
|
|
1647
|
+
yield
|
|
1648
|
+
if force_submit or not self._txn_.get().flatten:
|
|
1649
|
+
if qls := self._txn_.get().qls.pop():
|
|
1650
|
+
await self._execute(qls)
|
|
1651
|
+
finally:
|
|
1652
|
+
self._txn_.get().in_txn.pop()
|
|
1653
|
+
self._txn_.get().flatten = bak_flatten
|
|
1654
|
+
|
|
1655
|
+
@contextmanager
|
|
1656
|
+
def with_globals(self, globals_):
|
|
1657
|
+
if not self.direct_access:
|
|
1658
|
+
try:
|
|
1659
|
+
yield
|
|
1660
|
+
finally:
|
|
1661
|
+
raise NotImplemented('非直连模式不支持设置state信息')
|
|
1662
|
+
else:
|
|
1663
|
+
bak_cli = self.client
|
|
1664
|
+
try:
|
|
1665
|
+
self.client = self.client.with_globals(**globals_)
|
|
1666
|
+
yield
|
|
1667
|
+
finally:
|
|
1668
|
+
self.client = bak_cli
|
|
1669
|
+
|
|
1670
|
+
@contextmanager
|
|
1671
|
+
def without_globals(self, *global_names):
|
|
1672
|
+
if not self.direct_access:
|
|
1673
|
+
try:
|
|
1674
|
+
yield
|
|
1675
|
+
finally:
|
|
1676
|
+
raise NotImplemented('非直连模式不支持设置state信息')
|
|
1677
|
+
else:
|
|
1678
|
+
bak_cli = self.client
|
|
1679
|
+
try:
|
|
1680
|
+
self.client = self.client.without_globals(*global_names)
|
|
1681
|
+
yield
|
|
1682
|
+
finally:
|
|
1683
|
+
self.client = bak_cli
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
|
|
1687
|
+
synchronize = (
|
|
1688
|
+
'query_object', 'query', 'query_df',
|
|
1689
|
+
'execute', 'get_object',
|
|
1690
|
+
'insert_df', 'insert_df_pg'
|
|
1691
|
+
)
|
|
1692
|
+
|
|
1693
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
1694
|
+
def query_object(self, ql: str, **kwargs) -> List[Any]:
|
|
1695
|
+
...
|
|
1696
|
+
|
|
1697
|
+
def query(self, ql: str, **kwargs) -> List[Any]:
|
|
1698
|
+
...
|
|
1699
|
+
|
|
1700
|
+
def query_df(self, ql: str, **kwargs) -> pd.DataFrame:
|
|
1701
|
+
...
|
|
1702
|
+
|
|
1703
|
+
def execute(
|
|
1704
|
+
self,
|
|
1705
|
+
qls: Union[str, List[str], List[QueryWithArgs]],
|
|
1706
|
+
**kwargs
|
|
1707
|
+
) -> Optional[List]:
|
|
1708
|
+
...
|
|
1709
|
+
|
|
1710
|
+
def get_object(
|
|
1711
|
+
self,
|
|
1712
|
+
object_name: str,
|
|
1713
|
+
raw: bool = True
|
|
1714
|
+
) -> Optional[Union[ObjectTypeFrame, edgedb.Object]]:
|
|
1715
|
+
...
|
|
1716
|
+
|
|
1717
|
+
def insert_df(
|
|
1718
|
+
self,
|
|
1719
|
+
object_name: str,
|
|
1720
|
+
data: pd.DataFrame,
|
|
1721
|
+
relation: Dict[str, pd.DataFrame] = None,
|
|
1722
|
+
chunksize: int = 500,
|
|
1723
|
+
enable_upsert: bool = False,
|
|
1724
|
+
update_fields: Iterable[str] = None,
|
|
1725
|
+
exclusive_fields: Iterable[str] = None,
|
|
1726
|
+
) -> None:
|
|
1727
|
+
...
|
|
1728
|
+
|
|
1729
|
+
def insert_df_pg(
|
|
1730
|
+
self,
|
|
1731
|
+
object_name: str,
|
|
1732
|
+
data: pd.DataFrame,
|
|
1733
|
+
chunk_size: int = 500,
|
|
1734
|
+
enable_upsert: bool = False,
|
|
1735
|
+
exclusive_fields: List[str] = None,
|
|
1736
|
+
update_fields: List[str] = None,
|
|
1737
|
+
) -> None:
|
|
1738
|
+
...
|
|
1739
|
+
|
|
1740
|
+
@contextmanager
|
|
1741
|
+
def start_transaction(self, flatten: bool = False):
|
|
1742
|
+
"""开启事务
|
|
1743
|
+
|
|
1744
|
+
上下文管理器,使用with语法开启上下文,上下文中的ql将作为事务执行
|
|
1745
|
+
退出with语句块后,事务将立即执行,执行过程中如果报错会直接抛出
|
|
1746
|
+
|
|
1747
|
+
.. admonition:: 示例
|
|
1748
|
+
|
|
1749
|
+
.. code-block:: python
|
|
1750
|
+
|
|
1751
|
+
import pandas as pd
|
|
1752
|
+
|
|
1753
|
+
dm = DeepModel()
|
|
1754
|
+
|
|
1755
|
+
data = pd.DataFrame(
|
|
1756
|
+
{
|
|
1757
|
+
'name': ['Alice', 'Bob', 'Carol'],
|
|
1758
|
+
'deck': [
|
|
1759
|
+
"Dragon",
|
|
1760
|
+
"Golem",
|
|
1761
|
+
"Imp"
|
|
1762
|
+
],
|
|
1763
|
+
'awards': [
|
|
1764
|
+
"1st",
|
|
1765
|
+
"2nd",
|
|
1766
|
+
"3rd"
|
|
1767
|
+
],
|
|
1768
|
+
}
|
|
1769
|
+
)
|
|
1770
|
+
|
|
1771
|
+
with dm.start_transaction():
|
|
1772
|
+
dm.execute("delete User")
|
|
1773
|
+
dm.insert_df("User", data)
|
|
1774
|
+
|
|
1775
|
+
|
|
1776
|
+
Important:
|
|
1777
|
+
|
|
1778
|
+
仅 :func:`insert_df` :func:`execute` 方法支持在事务中执行
|
|
1779
|
+
|
|
1780
|
+
"""
|
|
1781
|
+
try:
|
|
1782
|
+
self._txn_.get()
|
|
1783
|
+
except LookupError:
|
|
1784
|
+
self._txn_.set(_TxnConfig())
|
|
1785
|
+
|
|
1786
|
+
bak_flatten = self._txn_.get().flatten
|
|
1787
|
+
self._txn_.get().in_txn.append(True)
|
|
1788
|
+
|
|
1789
|
+
if flatten and not self._txn_.get().flatten:
|
|
1790
|
+
force_submit = True
|
|
1791
|
+
else:
|
|
1792
|
+
force_submit = False
|
|
1793
|
+
|
|
1794
|
+
self._txn_.get().flatten = bak_flatten or flatten
|
|
1795
|
+
|
|
1796
|
+
if not self._txn_.get().flatten:
|
|
1797
|
+
self._txn_.get().qls.append([])
|
|
1798
|
+
|
|
1799
|
+
try:
|
|
1800
|
+
yield
|
|
1801
|
+
if force_submit or not self._txn_.get().flatten:
|
|
1802
|
+
if qls := self._txn_.get().qls.pop():
|
|
1803
|
+
evloop.run(self._execute(qls))
|
|
1804
|
+
finally:
|
|
1805
|
+
self._txn_.get().in_txn.pop()
|
|
1806
|
+
self._txn_.get().flatten = bak_flatten
|