deepfos 1.1.60__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. deepfos/__init__.py +6 -0
  2. deepfos/_version.py +21 -0
  3. deepfos/algo/__init__.py +0 -0
  4. deepfos/algo/graph.py +171 -0
  5. deepfos/algo/segtree.py +31 -0
  6. deepfos/api/V1_1/__init__.py +0 -0
  7. deepfos/api/V1_1/business_model.py +119 -0
  8. deepfos/api/V1_1/dimension.py +599 -0
  9. deepfos/api/V1_1/models/__init__.py +0 -0
  10. deepfos/api/V1_1/models/business_model.py +1033 -0
  11. deepfos/api/V1_1/models/dimension.py +2768 -0
  12. deepfos/api/V1_2/__init__.py +0 -0
  13. deepfos/api/V1_2/dimension.py +285 -0
  14. deepfos/api/V1_2/models/__init__.py +0 -0
  15. deepfos/api/V1_2/models/dimension.py +2923 -0
  16. deepfos/api/__init__.py +0 -0
  17. deepfos/api/account.py +167 -0
  18. deepfos/api/accounting_engines.py +147 -0
  19. deepfos/api/app.py +626 -0
  20. deepfos/api/approval_process.py +198 -0
  21. deepfos/api/base.py +983 -0
  22. deepfos/api/business_model.py +160 -0
  23. deepfos/api/consolidation.py +129 -0
  24. deepfos/api/consolidation_process.py +106 -0
  25. deepfos/api/datatable.py +341 -0
  26. deepfos/api/deep_pipeline.py +61 -0
  27. deepfos/api/deepconnector.py +36 -0
  28. deepfos/api/deepfos_task.py +92 -0
  29. deepfos/api/deepmodel.py +188 -0
  30. deepfos/api/dimension.py +486 -0
  31. deepfos/api/financial_model.py +319 -0
  32. deepfos/api/journal_model.py +119 -0
  33. deepfos/api/journal_template.py +132 -0
  34. deepfos/api/memory_financial_model.py +98 -0
  35. deepfos/api/models/__init__.py +3 -0
  36. deepfos/api/models/account.py +483 -0
  37. deepfos/api/models/accounting_engines.py +756 -0
  38. deepfos/api/models/app.py +1338 -0
  39. deepfos/api/models/approval_process.py +1043 -0
  40. deepfos/api/models/base.py +234 -0
  41. deepfos/api/models/business_model.py +805 -0
  42. deepfos/api/models/consolidation.py +711 -0
  43. deepfos/api/models/consolidation_process.py +248 -0
  44. deepfos/api/models/datatable_mysql.py +427 -0
  45. deepfos/api/models/deep_pipeline.py +55 -0
  46. deepfos/api/models/deepconnector.py +28 -0
  47. deepfos/api/models/deepfos_task.py +386 -0
  48. deepfos/api/models/deepmodel.py +308 -0
  49. deepfos/api/models/dimension.py +1576 -0
  50. deepfos/api/models/financial_model.py +1796 -0
  51. deepfos/api/models/journal_model.py +341 -0
  52. deepfos/api/models/journal_template.py +854 -0
  53. deepfos/api/models/memory_financial_model.py +478 -0
  54. deepfos/api/models/platform.py +178 -0
  55. deepfos/api/models/python.py +221 -0
  56. deepfos/api/models/reconciliation_engine.py +411 -0
  57. deepfos/api/models/reconciliation_report.py +161 -0
  58. deepfos/api/models/role_strategy.py +884 -0
  59. deepfos/api/models/smartlist.py +237 -0
  60. deepfos/api/models/space.py +1137 -0
  61. deepfos/api/models/system.py +1065 -0
  62. deepfos/api/models/variable.py +463 -0
  63. deepfos/api/models/workflow.py +946 -0
  64. deepfos/api/platform.py +199 -0
  65. deepfos/api/python.py +90 -0
  66. deepfos/api/reconciliation_engine.py +181 -0
  67. deepfos/api/reconciliation_report.py +64 -0
  68. deepfos/api/role_strategy.py +234 -0
  69. deepfos/api/smartlist.py +69 -0
  70. deepfos/api/space.py +582 -0
  71. deepfos/api/system.py +372 -0
  72. deepfos/api/variable.py +154 -0
  73. deepfos/api/workflow.py +264 -0
  74. deepfos/boost/__init__.py +6 -0
  75. deepfos/boost/py_jstream.py +89 -0
  76. deepfos/boost/py_pandas.py +20 -0
  77. deepfos/cache.py +121 -0
  78. deepfos/config.py +6 -0
  79. deepfos/core/__init__.py +27 -0
  80. deepfos/core/cube/__init__.py +10 -0
  81. deepfos/core/cube/_base.py +462 -0
  82. deepfos/core/cube/constants.py +21 -0
  83. deepfos/core/cube/cube.py +408 -0
  84. deepfos/core/cube/formula.py +707 -0
  85. deepfos/core/cube/syscube.py +532 -0
  86. deepfos/core/cube/typing.py +7 -0
  87. deepfos/core/cube/utils.py +238 -0
  88. deepfos/core/dimension/__init__.py +11 -0
  89. deepfos/core/dimension/_base.py +506 -0
  90. deepfos/core/dimension/dimcreator.py +184 -0
  91. deepfos/core/dimension/dimension.py +472 -0
  92. deepfos/core/dimension/dimexpr.py +271 -0
  93. deepfos/core/dimension/dimmember.py +155 -0
  94. deepfos/core/dimension/eledimension.py +22 -0
  95. deepfos/core/dimension/filters.py +99 -0
  96. deepfos/core/dimension/sysdimension.py +168 -0
  97. deepfos/core/logictable/__init__.py +5 -0
  98. deepfos/core/logictable/_cache.py +141 -0
  99. deepfos/core/logictable/_operator.py +663 -0
  100. deepfos/core/logictable/nodemixin.py +673 -0
  101. deepfos/core/logictable/sqlcondition.py +609 -0
  102. deepfos/core/logictable/tablemodel.py +497 -0
  103. deepfos/db/__init__.py +36 -0
  104. deepfos/db/cipher.py +660 -0
  105. deepfos/db/clickhouse.py +191 -0
  106. deepfos/db/connector.py +195 -0
  107. deepfos/db/daclickhouse.py +171 -0
  108. deepfos/db/dameng.py +101 -0
  109. deepfos/db/damysql.py +189 -0
  110. deepfos/db/dbkits.py +358 -0
  111. deepfos/db/deepengine.py +99 -0
  112. deepfos/db/deepmodel.py +82 -0
  113. deepfos/db/deepmodel_kingbase.py +83 -0
  114. deepfos/db/edb.py +214 -0
  115. deepfos/db/gauss.py +83 -0
  116. deepfos/db/kingbase.py +83 -0
  117. deepfos/db/mysql.py +184 -0
  118. deepfos/db/oracle.py +131 -0
  119. deepfos/db/postgresql.py +192 -0
  120. deepfos/db/sqlserver.py +99 -0
  121. deepfos/db/utils.py +135 -0
  122. deepfos/element/__init__.py +89 -0
  123. deepfos/element/accounting.py +348 -0
  124. deepfos/element/apvlprocess.py +215 -0
  125. deepfos/element/base.py +398 -0
  126. deepfos/element/bizmodel.py +1269 -0
  127. deepfos/element/datatable.py +2467 -0
  128. deepfos/element/deep_pipeline.py +186 -0
  129. deepfos/element/deepconnector.py +59 -0
  130. deepfos/element/deepmodel.py +1806 -0
  131. deepfos/element/dimension.py +1254 -0
  132. deepfos/element/fact_table.py +427 -0
  133. deepfos/element/finmodel.py +1485 -0
  134. deepfos/element/journal.py +840 -0
  135. deepfos/element/journal_template.py +943 -0
  136. deepfos/element/pyscript.py +412 -0
  137. deepfos/element/reconciliation.py +553 -0
  138. deepfos/element/rolestrategy.py +243 -0
  139. deepfos/element/smartlist.py +457 -0
  140. deepfos/element/variable.py +756 -0
  141. deepfos/element/workflow.py +560 -0
  142. deepfos/exceptions/__init__.py +239 -0
  143. deepfos/exceptions/hook.py +86 -0
  144. deepfos/lazy.py +104 -0
  145. deepfos/lazy_import.py +84 -0
  146. deepfos/lib/__init__.py +0 -0
  147. deepfos/lib/_javaobj.py +366 -0
  148. deepfos/lib/asynchronous.py +879 -0
  149. deepfos/lib/concurrency.py +107 -0
  150. deepfos/lib/constant.py +39 -0
  151. deepfos/lib/decorator.py +310 -0
  152. deepfos/lib/deepchart.py +778 -0
  153. deepfos/lib/deepux.py +477 -0
  154. deepfos/lib/discovery.py +273 -0
  155. deepfos/lib/edb_lexer.py +789 -0
  156. deepfos/lib/eureka.py +156 -0
  157. deepfos/lib/filterparser.py +751 -0
  158. deepfos/lib/httpcli.py +106 -0
  159. deepfos/lib/jsonstreamer.py +80 -0
  160. deepfos/lib/msg.py +394 -0
  161. deepfos/lib/nacos.py +225 -0
  162. deepfos/lib/patch.py +92 -0
  163. deepfos/lib/redis.py +241 -0
  164. deepfos/lib/serutils.py +181 -0
  165. deepfos/lib/stopwatch.py +99 -0
  166. deepfos/lib/subtask.py +572 -0
  167. deepfos/lib/sysutils.py +703 -0
  168. deepfos/lib/utils.py +1003 -0
  169. deepfos/local.py +160 -0
  170. deepfos/options.py +670 -0
  171. deepfos/translation.py +237 -0
  172. deepfos-1.1.60.dist-info/METADATA +33 -0
  173. deepfos-1.1.60.dist-info/RECORD +175 -0
  174. deepfos-1.1.60.dist-info/WHEEL +5 -0
  175. deepfos-1.1.60.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1806 @@
1
+ import re
2
+
3
+ import numpy as np
4
+ from asyncpg.connection import connect as pg_conn
5
+ import json
6
+ import uuid
7
+ from contextlib import asynccontextmanager, contextmanager
8
+ from contextvars import ContextVar
9
+ from itertools import count
10
+ from typing import (
11
+ List, TYPE_CHECKING, Any, Dict, Union, NamedTuple,
12
+ Iterable, Optional, Literal
13
+ )
14
+
15
+ import edgedb
16
+ import pandas as pd
17
+ from loguru import logger
18
+ from pydantic import BaseModel, parse_obj_as, ValidationError, Field
19
+
20
+ from deepfos import OPTION
21
+ from deepfos.api.deepmodel import DeepModelAPI
22
+ from deepfos.api.models.deepmodel import (
23
+ ObjectBasicDTO, ObjectParam,
24
+ QueryResultObjectInfo, QueryResult
25
+ )
26
+ from deepfos.cache import SpaceSeperatedLRUCache
27
+ from deepfos.db.edb import create_async_client
28
+ from deepfos.element.base import ElementBase, SyncMeta
29
+ from deepfos.exceptions import (
30
+ RequiredFieldUnfilled, ObjectNotExist,
31
+ ExternalObjectReadOnly, MultiLinkTargetNotUnique,
32
+ SingleLinkInRelation
33
+ )
34
+ from deepfos.lib import serutils
35
+ from deepfos.lib.asynchronous import future_property, evloop
36
+ from deepfos.lib.decorator import flagmethod, cached_property, lru_cache
37
+ from deepfos.lib.utils import AliasGenerator, to_version_tuple
38
+
39
+ __all__ = ['AsyncDeepModel', 'DeepModel', 'to_fields', 'QueryWithArgs']
40
+
41
+ OBJECT_QUERY = \
42
+ """
43
+ with module schema
44
+ select ObjectType {
45
+ id,
46
+ name,
47
+ links: {
48
+ id,
49
+ name,
50
+ cardinality,
51
+ required,
52
+ properties: { id, name, target: {name} } filter .name not in {'source', 'target'},
53
+ target: { id, name, external, annotations: {name, value := @value},
54
+ properties: { id, name } filter .name != 'id' },
55
+ expr,
56
+ constraints: { name, expr, finalexpr, subjectexpr },
57
+ source_property: {name},
58
+ target_property: {name},
59
+ annotations: {name, value := @value},
60
+ } filter .name != '__type__',
61
+ properties: {
62
+ id,
63
+ name,
64
+ cardinality,
65
+ required,
66
+ target: { name },
67
+ expr,
68
+ constraints: { name, expr, finalexpr, subjectexpr },
69
+ annotations: {name, value := @value},
70
+ } filter .name != 'id',
71
+ annotations: {name, value := @value},
72
+ constraints: { name, expr, finalexpr, subjectexpr },
73
+ external
74
+ }
75
+ """
76
+ BUSINESS_KEY = 'business_key'
77
+ BATCH_INSERT_KW = 'data'
78
+
79
+ # pandas to_json精度最大为15
80
+ # 为避免精度缺失,如下类型需转换为string
81
+ NEED_CAST_STR = ['std::decimal']
82
+
83
+ DOC_ARGS_KWARGS = """
84
+ Hint:
85
+
86
+ kwargs语法:
87
+
88
+ select User{name, is_active}
89
+ filter .name=<std::str>$name and is_active=<std::bool>$active
90
+
91
+ .. admonition:: 使用示例
92
+
93
+ .. code-block:: python
94
+
95
+ dm = DeepModel()
96
+
97
+ dm.execute(
98
+ '''delete User filter .name=<std::str>$name
99
+ and is_active=<std::bool>$active''',
100
+ name='Alice', active='True'
101
+ )
102
+
103
+ 此处 `$` 为以kwargs的方式指定参数的特殊符号,
104
+ 且需在参数前增加相应类型提示,参数值只支持str和int类型
105
+ """
106
+
107
+ NOT_SCALAR = "default::object"
108
+
109
+ dm_type_to_edb_scalar = {
110
+ 'str': 'std::str',
111
+ 'int': 'std::int64',
112
+ 'bool': 'std::bool',
113
+ 'multilingual': 'std::str',
114
+ 'float': 'std::decimal',
115
+ 'datetime': 'cal::local_datetime',
116
+ 'file': 'std::str',
117
+ 'uuid': 'std::str',
118
+ 'json': 'std::json',
119
+ }
120
+
121
+
122
+ class ObjectElement(ObjectParam):
123
+ @property
124
+ def links(self):
125
+ return {link.code: link for link in self.linkParamList}
126
+
127
+
128
+ class QueryWithArgs(BaseModel):
129
+ commands: str
130
+ kwargs: Dict[str, Any] = Field(default_factory=dict)
131
+
132
+
133
+ class MainField(NamedTuple):
134
+ business_key: str
135
+ is_multi: bool
136
+ props: Iterable[Optional[str]]
137
+ # 目前业务主键创建的类型只会为std::str
138
+ type: str = 'std::str'
139
+
140
+
141
+ class MainPgCol(NamedTuple):
142
+ target_bkey_col: str
143
+ target_col: str
144
+
145
+
146
+ class ConstraintField(BaseModel):
147
+ name: str
148
+ expr: str
149
+ finalexpr: str = None
150
+ subjectexpr: str = None
151
+
152
+
153
+ class NamedField(BaseModel):
154
+ name: str = None
155
+
156
+
157
+ class NameIdField(BaseModel):
158
+ id: uuid.UUID = None
159
+ name: str = None
160
+
161
+
162
+ class TargetField(BaseModel):
163
+ id: uuid.UUID = None
164
+ name: str
165
+ external: bool = False
166
+ annotations: List[Dict[str, str]] = Field(default_factory=list)
167
+ properties: List[NameIdField] = None
168
+
169
+ @property
170
+ def is_scalar(self) -> bool:
171
+ return self.name.startswith('std::') or self.name == 'cal::local_datetime'
172
+
173
+ @property
174
+ def info(self):
175
+ return {e['name'].rpartition('::')[-1]: e['value'] for e in self.annotations}
176
+
177
+ @property
178
+ def normalized_name(self):
179
+ return self.name.rpartition('::')[-1]
180
+
181
+ @property
182
+ def props(self):
183
+ return {p.name: p for p in self.properties}
184
+
185
+
186
+ class LinkPropField(BaseModel):
187
+ id: uuid.UUID = None
188
+ name: str
189
+ target: NamedField
190
+
191
+ @property
192
+ def type(self) -> str:
193
+ return self.target.name
194
+
195
+
196
+ class PtrInfo(BaseModel):
197
+ id: Union[uuid.UUID, Literal['id']] = None
198
+ name: str
199
+ target: TargetField
200
+ properties: List[LinkPropField] = Field(default_factory=list)
201
+ expr: str = None
202
+ source_property: NamedField = None
203
+ target_property: NamedField = None
204
+ required: bool = False
205
+ cardinality: str = None
206
+ constraints: List[ConstraintField] = Field(default_factory=list)
207
+ annotations: List[Dict[str, str]] = Field(default_factory=list)
208
+
209
+ @property
210
+ def type(self) -> str:
211
+ return self.target.name
212
+
213
+ @property
214
+ def is_link(self):
215
+ return not self.target.is_scalar
216
+
217
+ @property
218
+ def is_multi_link(self):
219
+ return self.is_link and self.cardinality == 'Many'
220
+
221
+ @property
222
+ def is_multi(self):
223
+ return self.cardinality == 'Many'
224
+
225
+ @property
226
+ def computable(self):
227
+ return self.expr is not None
228
+
229
+ @property
230
+ def external(self):
231
+ return self.target.external
232
+
233
+ @property
234
+ def props(self):
235
+ return [p.name for p in self.properties]
236
+
237
+ @property
238
+ def prop_type(self):
239
+ return {p.name: p.type for p in self.properties}
240
+
241
+ @property
242
+ def target_col(self):
243
+ if self.target_property:
244
+ return self.target_property.name
245
+
246
+ @property
247
+ def info(self):
248
+ return {e['name'].rpartition('::')[-1]: e['value'] for e in self.annotations}
249
+
250
+
251
+ _RE_CONSTRAINT_FIELDS = re.compile(r'\((((\.\w+)(,\s+)?)+)\)')
252
+
253
+
254
+ class ObjectTypeFrame(BaseModel):
255
+ id: uuid.UUID = None
256
+ name: str
257
+ links: List[PtrInfo] = Field(default_factory=list)
258
+ properties: List[PtrInfo] = Field(default_factory=list)
259
+ external: bool
260
+ annotations: List[Dict[str, str]] = Field(default_factory=list)
261
+ constraints: List[ConstraintField] = Field(default_factory=list)
262
+
263
+ @property
264
+ def fields(self):
265
+ return {ptr.name: ptr for ptr in [*self.links, *self.properties]}
266
+
267
+ @property
268
+ def info(self):
269
+ return {e['name'].rpartition('::')[-1]: e['value'] for e in self.annotations}
270
+
271
+ @property
272
+ def normalized_name(self):
273
+ return self.name.rpartition('::')[-1]
274
+
275
+ @property
276
+ def exclusive_fields(self):
277
+ exclusive = [
278
+ field.name
279
+ for field in self.fields.values()
280
+ if field.name != 'id' and any([
281
+ const.expr and (
282
+ const.name == 'std::exclusive'
283
+ and const.subjectexpr is None
284
+ )
285
+ for const in field.constraints
286
+ ])
287
+ ]
288
+ for const in self.constraints:
289
+ if (
290
+ const.name == 'std::exclusive'
291
+ and const.subjectexpr and (
292
+ m := _RE_CONSTRAINT_FIELDS.match(const.subjectexpr)
293
+ )
294
+ ):
295
+ found = m.group(1)
296
+ exclusive.append({f.strip()[1:] for f in found.split(',')})
297
+ return exclusive
298
+
299
+
300
+ def _format_link(df: pd.DataFrame, link_name: str):
301
+ if all(pd.isnull(df['target'])):
302
+ return {'target': pd.NA}
303
+
304
+ record = df.drop(columns=['source']).set_index('target')
305
+
306
+ if not record.index.is_unique:
307
+ raise MultiLinkTargetNotUnique(
308
+ f'Multi Link: [{link_name}] relation dataframe中'
309
+ f'source与target对应存在不唯一性'
310
+ )
311
+
312
+ record = record.to_dict(orient='index')
313
+ return {'prop': record, 'target': list(record.keys())}
314
+
315
+
316
+ class BaseField(PtrInfo):
317
+ def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
318
+ """使 :class:`Dataframe` 对应的列符合字段的限制条件
319
+
320
+ Args:
321
+ df: 待转换的 :class:`Dataframe`
322
+ field_name: 需要转化的列名
323
+ raw_pg: 是否在为pg插入做fit
324
+ """
325
+ pass
326
+
327
+ def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
328
+ """
329
+ 对 :class:`Dataframe` 对应的列作类型转换。
330
+ 一般在获取 :class:`Dataframe` 时使用。
331
+ """
332
+ pass
333
+
334
+
335
+ class FieldDateTime(BaseField):
336
+ @staticmethod
337
+ def format_datetime(dt):
338
+ if pd.isna(dt):
339
+ return pd.NA
340
+ return pd.to_datetime(dt).strftime("%Y-%m-%dT%H:%M:%S")
341
+
342
+ def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
343
+ df[field_name] = df[field_name].apply(self.format_datetime)
344
+
345
+ def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
346
+ df[field_name] = pd.to_datetime(df[field_name], errors='ignore')
347
+
348
+
349
+ class FieldString(BaseField):
350
+ def format_string(self, data):
351
+ if pd.isna(data):
352
+ return pd.NA
353
+ return str(data)
354
+
355
+ def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
356
+ df[field_name] = df[field_name].apply(self.format_string)
357
+
358
+
359
+ class FieldJson(BaseField):
360
+ def format_json(self, data):
361
+ if pd.isna(data):
362
+ return data
363
+ return json.dumps(data)
364
+
365
+ def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
366
+ # std::json needed to be fit only when data is for pg batch insert
367
+ # because json-object (dict in pd series) will be encoded correctly
368
+ # by pandas to_json
369
+ if raw_pg:
370
+ df[field_name] = df[field_name].apply(self.format_json)
371
+
372
+ def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
373
+ # std::json needed to be cast only when data is from http
374
+ # since json value will be converted to json string(type: str)
375
+ # in edgedb python protocol
376
+ if not direct_access:
377
+ df[field_name] = df[field_name].apply(self.format_json)
378
+
379
+
380
+ class FieldInt(BaseField):
381
+ def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
382
+ df[field_name] = pd.to_numeric(df[field_name])
383
+
384
+ def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
385
+ df[field_name] = df[field_name].astype(pd.Int64Dtype(), errors='ignore')
386
+
387
+
388
+ class FieldDecimal(FieldString):
389
+ def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
390
+ df[field_name] = df[field_name].astype(float, errors='ignore')
391
+
392
+
393
+ class FieldBool(BaseField):
394
+ def format_bool(self, data):
395
+ if isinstance(data, bool) or pd.isna(data):
396
+ return data
397
+ return str(data).lower() == 'true'
398
+
399
+ def fit(self, df: pd.DataFrame, field_name: str, raw_pg: bool = False):
400
+ df[field_name] = df[field_name].apply(self.format_bool)
401
+
402
+ def cast(self, df: pd.DataFrame, field_name: str, direct_access: bool = True):
403
+ pass
404
+
405
+
406
+ class FieldFactory:
407
+ field_map = {
408
+ 'std::bool': FieldBool,
409
+ 'std::int64': FieldInt,
410
+ 'std::bigint': FieldInt,
411
+ 'std::decimal': FieldDecimal,
412
+ 'std::json': FieldJson,
413
+ 'std::str': FieldString,
414
+ 'cal::local_datetime': FieldDateTime,
415
+ }
416
+
417
+ def __new__(cls, field: Union[PtrInfo, LinkPropField]) -> BaseField:
418
+ field_class = cls.field_map.get(field.type, BaseField)
419
+ return field_class(**field.dict())
420
+
421
+
422
+ class ObjectStructure:
423
+ fields: Dict[str, BaseField]
424
+
425
+ def __init__(self, name, structure: Iterable[PtrInfo], include_id=False):
426
+ self.name = name
427
+ self.fields = {
428
+ field.name: FieldFactory(field)
429
+ for field in structure
430
+ if not field.computable and field.name != 'id'
431
+ }
432
+ # Make one for batch insert pg
433
+ if include_id:
434
+ self.fields['id'] = BaseField(
435
+ name='id', id='id', cardinality='One',
436
+ target=TargetField(name='std::uuid')
437
+ )
438
+ self.self_link_fields = []
439
+ for name, field in list(self.fields.items()):
440
+ if field.type == self.name:
441
+ self.self_link_fields.append(name)
442
+ if field.is_multi_link:
443
+ continue
444
+ if not field.is_link:
445
+ continue
446
+ for prop in field.properties:
447
+ self.fields[f'{name}@{prop.name}'] = FieldFactory(prop)
448
+
449
+ def fit(self, df: pd.DataFrame, raw_pg=False) -> pd.DataFrame:
450
+ """
451
+ 对传入的DataFrame的指定数据列执行fit操作。
452
+ 直接影响DataFrame数据。
453
+
454
+ Args:
455
+ df: 数据源
456
+ raw_pg: 是否在为pg插入做fit
457
+
458
+ """
459
+ valid_fields = []
460
+ for field in df.columns:
461
+ if field in self.fields:
462
+ valid_fields.append(field)
463
+ self.fields[field].fit(df, field, raw_pg)
464
+
465
+ return df[valid_fields]
466
+
467
+ def cast(self, df: pd.DataFrame, direct_access: bool = True):
468
+ for field in df.columns:
469
+ if field in self.fields:
470
+ self.fields[field].cast(df, field, direct_access)
471
+
472
+
473
+ def _iter_link_prop_assign(link, business_key, prop_name, prop_type, is_multi):
474
+ assign_string = f"@{prop_name} := <{prop_type}>"
475
+ if prop_type in NEED_CAST_STR:
476
+ assign_string += '<std::str>'
477
+ if is_multi:
478
+ return f"{assign_string}(json_get(item, '{link}', 'prop', .{business_key}, '{prop_name}'))"
479
+
480
+ return f"{assign_string}(json_get(item, '{link}@{prop_name}'))"
481
+
482
+
483
+ def _iter_single_assign(
484
+ field: PtrInfo,
485
+ cast_type: str,
486
+ target_main_field: Dict[str, MainField]
487
+ ):
488
+ assign_string = f"{field.name} := "
489
+ # 设置标量值
490
+ if field.name not in target_main_field:
491
+ assign_string += f"<{cast_type}>"
492
+
493
+ if cast_type in NEED_CAST_STR:
494
+ assign_string += '<std::str>'
495
+
496
+ return assign_string + f"item['{field.name}']"
497
+
498
+ # 设置link target值
499
+ link = field.name
500
+ main_field = target_main_field[link]
501
+
502
+ if main_field.props:
503
+ target = (
504
+ cast_type + "{" +
505
+ ",".join(
506
+ _iter_link_prop_assign(link, main_field.business_key, name,
507
+ field.prop_type[name], main_field.is_multi)
508
+ for name in main_field.props
509
+ ) + "}"
510
+ )
511
+ else:
512
+ target = cast_type
513
+
514
+ if main_field.is_multi:
515
+ assign_string += f"""(
516
+ select detached {target}
517
+ filter contains(
518
+ <array<{main_field.type}>>(json_get(item, '{link}', 'target')),
519
+ .{main_field.business_key}
520
+ )
521
+ )"""
522
+ else:
523
+ assign_string += f"""assert_single((
524
+ select detached {target}
525
+ filter .{main_field.business_key} = <{main_field.type}>(json_get(item, '{link}'))
526
+ ))"""
527
+
528
+ return assign_string
529
+
530
+
531
+ def bulk_insert_by_fields(
532
+ object_name: str,
533
+ field_type: List[PtrInfo],
534
+ target_main_field: Dict[str, MainField],
535
+ ):
536
+ insert_assign_body = ','.join(
537
+ [
538
+ _iter_single_assign(field, field.type, target_main_field)
539
+ for field in field_type
540
+ ]
541
+ )
542
+
543
+ return f"""
544
+ with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
545
+ for item in json_array_unpack(raw_data) union (
546
+ insert {object_name} {{
547
+ {insert_assign_body}
548
+ }}
549
+ )
550
+ """
551
+
552
+
553
+ def bulk_upsert_by_fields(
554
+ object_name: str,
555
+ field_type: List[PtrInfo],
556
+ target_main_field: Dict[str, MainField],
557
+ exclusive_fields: Iterable[str],
558
+ update_fields: Iterable[str]
559
+ ):
560
+ conflict_on_fields = map(lambda n: f'.{n}', exclusive_fields)
561
+
562
+ insert_assign_body = ','.join(
563
+ [
564
+ _iter_single_assign(field, field.type, target_main_field)
565
+ for field in field_type
566
+ ]
567
+ )
568
+ update_assign_body = ','.join(
569
+ [
570
+ _iter_single_assign(field, field.type, target_main_field)
571
+ for field in field_type if field.name in update_fields
572
+ ]
573
+ )
574
+
575
+ return f"""
576
+ with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
577
+ for item in json_array_unpack(raw_data) union (
578
+ insert {object_name} {{
579
+ {insert_assign_body}
580
+ }}
581
+ unless conflict on ({','.join(conflict_on_fields)})
582
+ else (
583
+ update {object_name} set {{
584
+ {update_assign_body}
585
+ }}
586
+ )
587
+ )
588
+ """
589
+
590
+
591
+ def bulk_update_by_fields(
592
+ object_name: str,
593
+ business_key: str,
594
+ field_type: List[PtrInfo],
595
+ target_main_field: Dict[str, MainField],
596
+ update_fields: Iterable[str] = None,
597
+ ):
598
+ update_assign_body = ','.join(
599
+ [
600
+ _iter_single_assign(field, field.type, target_main_field)
601
+ for field in field_type if field.name in update_fields
602
+ ]
603
+ )
604
+
605
+ return f"""
606
+ with raw_data := <json>to_json(<std::str>${BATCH_INSERT_KW}),
607
+ for item in json_array_unpack(raw_data) union (
608
+ update {object_name}
609
+ filter .{business_key} = <std::str>item['{business_key}']
610
+ set {{
611
+ {update_assign_body}
612
+ }}
613
+ )
614
+ """
615
+
616
+
617
+ def format_obj(obj: edgedb.Object) -> ObjectTypeFrame:
618
+ if not isinstance(obj, edgedb.Object):
619
+ raise TypeError("预期obj为edgedb.Object")
620
+
621
+ serialized = serutils.serialize(obj)
622
+
623
+ try:
624
+ return parse_obj_as(ObjectTypeFrame, serialized)
625
+ except ValidationError:
626
+ raise TypeError("预期obj为ObjectType查询得到的结构信息")
627
+
628
+
629
+ def to_fields(obj: edgedb.Object) -> Dict[str, PtrInfo]:
630
+ return format_obj(obj).fields
631
+
632
+
633
+ def collect_query_result_structure(
634
+ object_info: QueryResultObjectInfo
635
+ ):
636
+ fields = [
637
+ PtrInfo(
638
+ name=f.name,
639
+ target=NamedField(name=dm_type_to_edb_scalar.get(f.type, NOT_SCALAR))
640
+ )
641
+ for f in object_info.fields
642
+ ]
643
+ return ObjectStructure(name='', structure=fields)
644
+
645
+
646
+ def collect_frame_desc_structure(desc: Dict[str, str]):
647
+ fields = [
648
+ PtrInfo(
649
+ name=name if isinstance(name, str) else name[0],
650
+ target=NamedField(
651
+ name=tname
652
+ if isinstance(tname, str) else NOT_SCALAR
653
+ )
654
+ )
655
+ for name, tname in desc.items()
656
+ ]
657
+ return ObjectStructure(name='', structure=fields)
658
+
659
+
660
+ # Copied from edb/pgsql/types.py base_type_name_map
661
+ base_type_name_map = {
662
+ 'std::str': ('text',),
663
+ 'std::int64': ('int8',),
664
+ 'std::decimal': ('numeric',),
665
+ 'std::bool': ('bool',),
666
+ # for id col only
667
+ 'std::uuid': ('uuid',),
668
+ 'std::json': ('jsonb',),
669
+ 'cal::local_datetime': ('edgedb', 'timestamp_t')
670
+ }
671
+
672
+
673
+ def quote_ident(string: Union[str, uuid.UUID]) -> str:
674
+ return '"' + str(string).replace('"', '""') + '"'
675
+
676
+
677
+ def pg_type_cast(edb_type: str):
678
+ if edb_type in base_type_name_map:
679
+ return '::' + '.'.join(base_type_name_map[edb_type])
680
+ return ''
681
+
682
+
683
+ def _iter_value(
684
+ field: PtrInfo,
685
+ idx: int,
686
+ target_info: Dict[str, MainPgCol],
687
+ alias: AliasGenerator
688
+ ):
689
+ if field.name not in target_info:
690
+ if field.type == 'std::decimal':
691
+ return f'/*{field.name}*/"edgedb"."str_to_decimal"(${idx})'
692
+ if field.type == 'cal::local_datetime':
693
+ return f'/*{field.name}*/"edgedb"."local_datetime_in"(${idx})'
694
+ return f'/*{field.name}*/${idx}{pg_type_cast(field.type)}'
695
+
696
+ main_col = target_info[field.name]
697
+ tmp_table = quote_ident(alias.get('t'))
698
+ bkey = quote_ident(main_col.target_bkey_col)
699
+ # bkey is std::str
700
+ return (f"/*{field.name}*/\n"
701
+ f"(select {tmp_table}.{quote_ident(main_col.target_col)} "
702
+ f"from edgedbpub.{quote_ident(field.target.id)} as {tmp_table} "
703
+ f"where {tmp_table}.{bkey} = ${idx}{pg_type_cast('std::str')})")
704
+
705
+
706
+ def batch_insert_pg(
707
+ object_id: str,
708
+ fields: List[PtrInfo],
709
+ target_info: Dict[str, MainPgCol],
710
+ exclusive_fields: List[PtrInfo] = None,
711
+ update_fields: List[PtrInfo] = None
712
+ ):
713
+ upd_fields = list(map(lambda f: quote_ident(f.id), update_fields or []))
714
+
715
+ alias = AliasGenerator()
716
+ idx = count(1).__next__
717
+ col_assign = {}
718
+ for field in fields:
719
+ col = quote_ident(field.id)
720
+ col_assign[col] = _iter_value(field, idx(), target_info, alias)
721
+
722
+ insert_main_tbl = (
723
+ f'INSERT INTO edgedbpub.{quote_ident(object_id)}'
724
+ f'({", ".join(col_assign.keys())})'
725
+ f' VALUES ({", ".join(col_assign.values())})'
726
+ )
727
+ if exclusive_fields and upd_fields:
728
+ conflicts = ','.join(quote_ident(ef.id) for ef in exclusive_fields)
729
+ insert_main_tbl += (
730
+ f' ON CONFLICT ({conflicts}) DO UPDATE SET ' +
731
+ ','.join([f'{col} = EXCLUDED.{col}' for col in upd_fields])
732
+ )
733
+ return insert_main_tbl
734
+
735
+
736
+ txn_support = flagmethod('_txn_support_')
737
+
738
+
739
+ class _TxnConfig:
740
+ __slots__ = ('qls', 'in_txn', 'txn_support', 'flatten')
741
+
742
+ def __init__(self):
743
+ self.qls = [[]]
744
+ self.in_txn = [False]
745
+ self.txn_support = False
746
+ self.flatten = False
747
+
748
+
749
+ @lru_cache(maxsize=128, cache_factory=SpaceSeperatedLRUCache)
750
+ async def get_element_info():
751
+ from deepfos.api.space import SpaceAPI
752
+ from deepfos.api.models.app import ElementRelationInfo
753
+ from deepfos.exceptions import ElementTypeMissingError
754
+ modules = await SpaceAPI(sync=False).module.get_usable_module()
755
+ target_module = ['MAINVIEW', 'DM']
756
+ for mdl in modules:
757
+ if mdl.moduleType in target_module and mdl.status == 1:
758
+ return ElementRelationInfo.construct_from(mdl)
759
+ raise ElementTypeMissingError('DeepModel组件在空间内不可用')
760
+
761
+
762
+ # -----------------------------------------------------------------------------
763
+ # core
764
+ class AsyncDeepModel(ElementBase[DeepModelAPI]):
765
+ """DeepModel元素
766
+
767
+ Args:
768
+ direct_access: 是否使用直连模式
769
+ pg_dsn: PG连接信息
770
+ """
771
+ __mangle_docs__ = False
772
+
773
+ def __init__(self, direct_access: bool = True, pg_dsn: str = None): # noqa
774
+ self._txn_ = ContextVar('QLTXN')
775
+ self.appmodule = f"app{OPTION.api.header['app']}"
776
+ self.spacemodule = f"space{OPTION.api.header['space']}"
777
+ self.direct_access = direct_access
778
+ self.alias = AliasGenerator()
779
+ self.pg_dsn = pg_dsn
780
+
781
+ @future_property
782
+ async def client(self):
783
+ if self.direct_access:
784
+ api = await self.wait_for('async_api')
785
+ ver = await api.extra.version()
786
+ if to_version_tuple(ver, -1) >= (3, 0, 18, 8, 0):
787
+ db_info = await api.sharding.database()
788
+ space = OPTION.api.header['space']
789
+ if db_info.space != space:
790
+ raise ValueError(
791
+ f'Space id in sharding database info invalid. '
792
+ f'Expected space id: {space}, actual: {db_info.space}'
793
+ )
794
+ dbname = db_info.edgedbName
795
+ else:
796
+ dbname = None
797
+ client = create_async_client(
798
+ default_module=self.appmodule,
799
+ dbname=dbname
800
+ )
801
+ if user_id := OPTION.api.header.get('user'):
802
+ client = client.with_globals(
803
+ **{
804
+ f'{self.spacemodule}::current_user_id':
805
+ user_id
806
+ }
807
+ )
808
+ return client
809
+
810
+ @future_property
811
+ async def element_info(self):
812
+ """元素信息"""
813
+ return await get_element_info()
814
+
815
+ @future_property
816
+ async def async_api(self):
817
+ """异步API对象"""
818
+ return await self._init_api()
819
+
820
+ def _safe_get_txn_conf(self) -> _TxnConfig:
821
+ try:
822
+ config = self._txn_.get()
823
+ except LookupError:
824
+ config = _TxnConfig()
825
+ self._txn_.set(config)
826
+ return config
827
+
828
+ @property
829
+ def _txn_support_(self):
830
+ return self._safe_get_txn_conf().txn_support
831
+
832
+ @_txn_support_.setter
833
+ def _txn_support_(self, val):
834
+ self._safe_get_txn_conf().txn_support = val
835
+
836
+ @future_property(on_demand=True)
837
+ async def model_objects(self) -> Dict[str, ObjectParam]:
838
+ """MainView中的所有对象详情"""
839
+ api = await self.wait_for('async_api')
840
+ res = await api.object.get_all()
841
+ return {obj.code: obj for obj in res.objectList}
842
+
843
+ @future_property(on_demand=True)
844
+ async def model_object_list(self) -> List[ObjectBasicDTO]:
845
+ """MainView中的所有对象列表"""
846
+ api = await self.wait_for('async_api')
847
+ return await api.object.list()
848
+
849
+ @future_property(on_demand=True)
850
+ async def user_objects(self) -> Dict[str, edgedb.Object]:
851
+ """当前app下所有的用户对象"""
852
+ objects = await AsyncDeepModel.query_object(
853
+ self,
854
+ f"{OBJECT_QUERY} filter .name like '{self.appmodule}::%'",
855
+ )
856
+ return {obj.name.rpartition('::')[-1]: obj for obj in objects}
857
+
858
+ @future_property(on_demand=True)
859
+ async def system_objects(self) -> Dict[str, edgedb.Object]:
860
+ """当前space下所有的系统对象"""
861
+ objects = await AsyncDeepModel.query_object(
862
+ self,
863
+ f"{OBJECT_QUERY} filter .name like '{self.spacemodule}::%'",
864
+ )
865
+ return {obj.name.rpartition('::')[-1]: obj for obj in objects}
866
+
867
+ @cached_property
868
+ def objects(self) -> Dict[str, ObjectTypeFrame]:
869
+ result = {name: format_obj(obj) for name, obj in self.user_objects.items()}
870
+ for name, obj in self.system_objects.items():
871
+ # In case of duplicate name both in app obj& space obj
872
+ if name in result:
873
+ result[f'{self.spacemodule}::{name}'] = format_obj(obj)
874
+ else:
875
+ result[name] = format_obj(obj)
876
+
877
+ return result
878
+
879
+ @staticmethod
880
+ def _prepare_variables(kwargs):
881
+ variables = {}
882
+ for k, v in kwargs.items():
883
+ variables[str(k)] = v
884
+ return variables
885
+
886
+ async def query_object(self, ql: str, **kwargs) -> List[Any]:
887
+ """执行ql查询语句,得到原始结果返回
888
+
889
+ 如有变量,以kwargs的方式提供
890
+
891
+ Args:
892
+ ql: 执行的ql
893
+
894
+ See Also:
895
+
896
+ :func:`query`, 执行ql查询语句,得到序列化后的结果
897
+ :func:`query_df`, 执行ql查询语句,获取DataFrame格式的二维表
898
+
899
+ """
900
+
901
+ if self.direct_access:
902
+ logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
903
+ _, result = await self.client.query(ql, **kwargs)
904
+ return result
905
+
906
+ result = await AsyncDeepModel.query(self, ql, **kwargs)
907
+ return serutils.deserialize(result)
908
+
909
+ async def query(self, ql: str, **kwargs) -> List[Any]:
910
+ """执行ql查询语句,得到序列化后的结果
911
+
912
+ 如有变量,以args, kwargs的方式提供
913
+
914
+ Args:
915
+ ql: 执行的ql
916
+
917
+
918
+ .. admonition:: 示例
919
+
920
+ .. code-block:: python
921
+
922
+ dm = DeepModel()
923
+
924
+ # 以变量name 查询User对象
925
+ dm.query(
926
+ 'select User{name, is_active} filter .name=<std::str>$name',
927
+ name='Alice'
928
+ )
929
+
930
+ See Also:
931
+
932
+ :func:`query_df`, 执行ql查询语句,获取DataFrame格式的二维表
933
+ :func:`query_object`, 执行ql查询语句,得到原始结果返回
934
+
935
+ """
936
+ if self.direct_access:
937
+ logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
938
+ frame_desc, result = await self.client.query(ql, **kwargs)
939
+ return serutils.serialize(
940
+ result, ctx=serutils.Context(frame_desc=frame_desc)
941
+ )
942
+
943
+ result = await self._http_query(ql, **kwargs)
944
+ return result.json_
945
+
946
+ async def _http_query(self, ql: str, **kwargs) -> QueryResult:
947
+ logger.opt(lazy=True).debug(f"Query: [{ql}], \nkwargs: [{kwargs}].")
948
+ result = await self.async_api.deepql.query(
949
+ module=self.appmodule,
950
+ query=ql,
951
+ variables=self._prepare_variables(kwargs)
952
+ )
953
+ self._maybe_handle_error(result.json_)
954
+ return result
955
+
956
+ async def query_df(self, ql: str, **kwargs) -> pd.DataFrame:
957
+ """执行ql查询语句
958
+
959
+ 获取DataFrame格式的二维表
960
+ 如有变量,以kwargs的方式提供
961
+
962
+ Args:
963
+ ql: 执行的ql
964
+
965
+
966
+ .. admonition:: 示例
967
+
968
+ .. code-block:: python
969
+
970
+ dm = DeepModel()
971
+
972
+ # 以变量name 查询User对象,得到DataFrame
973
+ dm.query_df(
974
+ 'select User{name, is_active} filter .name=<std::str>$name',
975
+ name='Alice'
976
+ )
977
+
978
+ See Also:
979
+
980
+ :func:`query`, 执行ql查询语句,得到序列化后的结果
981
+ :func:`query_object`, 执行ql查询语句,得到原始结果返回
982
+
983
+ """
984
+ if self.direct_access:
985
+ frame_desc, data = await self.client.query(ql, **kwargs)
986
+
987
+ data = pd.DataFrame(data=serutils.serialize(
988
+ data, ctx=serutils.Context(frame_desc=frame_desc, query_df=True)
989
+ ))
990
+ # Not records for dict-like
991
+ if not isinstance(frame_desc, dict):
992
+ return data
993
+
994
+ structure = collect_frame_desc_structure(frame_desc)
995
+ else:
996
+ result = await self._http_query(ql, **kwargs)
997
+ # No object structure info
998
+ if result.objectInfos is None:
999
+ return pd.DataFrame(data=result.json_)
1000
+
1001
+ data = pd.DataFrame(
1002
+ data=result.json_,
1003
+ columns=[f.name for f in result.objectInfos[0].fields]
1004
+ )
1005
+
1006
+ structure = collect_query_result_structure(result.objectInfos[0])
1007
+
1008
+ if data.empty:
1009
+ return pd.DataFrame(columns=structure.fields.keys())
1010
+
1011
+ structure.cast(data, self.direct_access)
1012
+ return data
1013
+
1014
+ query.__doc__ = query.__doc__ + DOC_ARGS_KWARGS
1015
+ query_object.__doc__ = query_object.__doc__ + DOC_ARGS_KWARGS
1016
+ query_df.__doc__ = query_df.__doc__ + DOC_ARGS_KWARGS
1017
+
1018
+ @txn_support
1019
+ async def execute(
1020
+ self,
1021
+ qls: Union[str, List[str], List[QueryWithArgs]],
1022
+ **kwargs
1023
+ ) -> Optional[List]:
1024
+ """以事务执行多句ql
1025
+
1026
+ Args:
1027
+ qls: 要执行的若干ql语句
1028
+ 可通过提供QueryWithArgs对象ql的方式定制每句ql的参数信息
1029
+ 亦可直接以kwargs的形式提供参数信息
1030
+ 会自动用作所有string形式ql的参数
1031
+
1032
+ """
1033
+ if isinstance(qls, str):
1034
+ qls_with_args = [QueryWithArgs(commands=qls, kwargs=kwargs)]
1035
+ else:
1036
+ qls_with_args = []
1037
+ seen_kwargs_key = set()
1038
+ for ql in qls:
1039
+ if isinstance(ql, QueryWithArgs):
1040
+ if (
1041
+ not self.direct_access
1042
+ and ql.kwargs
1043
+ and seen_kwargs_key.intersection(ql.kwargs.keys())
1044
+ ):
1045
+ raise NotImplementedError('非直连模式不支持重名variables')
1046
+
1047
+ qls_with_args.append(ql)
1048
+ if ql.kwargs:
1049
+ seen_kwargs_key = seen_kwargs_key.union(ql.kwargs.keys())
1050
+
1051
+ elif isinstance(ql, str):
1052
+ qls_with_args.append(QueryWithArgs(commands=ql, kwargs=kwargs))
1053
+ else:
1054
+ raise TypeError(f'qls参数中出现类型非法成员:{type(ql)}')
1055
+
1056
+ return await self._maybe_exec_qls(qls_with_args)
1057
+
1058
+ execute.__doc__ = execute.__doc__ + DOC_ARGS_KWARGS
1059
+
1060
+ async def _execute(self, qls_with_args: List[QueryWithArgs]) -> List:
1061
+ self.alias.reset(BATCH_INSERT_KW)
1062
+ if not self.direct_access:
1063
+ kwargs = {}
1064
+ seen_kwargs_key = set()
1065
+
1066
+ for ql in qls_with_args:
1067
+ if ql.kwargs and seen_kwargs_key.intersection(ql.kwargs.keys()):
1068
+ raise NotImplementedError('非直连模式不支持重名variables')
1069
+ if ql.kwargs:
1070
+ kwargs.update(ql.kwargs)
1071
+ seen_kwargs_key = seen_kwargs_key.union(ql.kwargs.keys())
1072
+
1073
+ commands = ';'.join([ql.commands for ql in qls_with_args])
1074
+
1075
+ logger.opt(lazy=True).debug(
1076
+ f"Execute QL: [{commands}], \nkwargs: [{kwargs}]."
1077
+ )
1078
+ res = await self.async_api.deepql.execute(
1079
+ module=self.appmodule,
1080
+ query=commands,
1081
+ variables=self._prepare_variables(kwargs)
1082
+ )
1083
+ affected = res.get('json')
1084
+ self._maybe_handle_error(affected)
1085
+ return affected
1086
+
1087
+ result = []
1088
+ async for tx in self.client.transaction():
1089
+ async with tx:
1090
+ for ql in qls_with_args:
1091
+ logger.opt(lazy=True).debug(
1092
+ f"Execute QL: [{ql.commands}], \nkwargs: [{ql.kwargs}]."
1093
+ )
1094
+ desc, affected = await tx.execute(ql.commands, **ql.kwargs)
1095
+ result.append(serutils.serialize(
1096
+ affected, ctx=serutils.Context(frame_desc=desc)
1097
+ ))
1098
+ if len(result) == 1:
1099
+ return result[0]
1100
+ return result
1101
+
1102
+ @staticmethod
1103
+ def _maybe_handle_error(res):
1104
+ if not isinstance(res, dict):
1105
+ return
1106
+
1107
+ if error := res.get('errors'): # pragma: no cover
1108
+ ex_msg = error['message'].strip()
1109
+ ex_code = int(error['code'])
1110
+ raise edgedb.EdgeDBError._from_code(ex_code, ex_msg) # noqa
1111
+
1112
+ async def _maybe_exec_qls(
1113
+ self,
1114
+ qls_with_args: List[QueryWithArgs]
1115
+ ) -> Optional[List]:
1116
+ txn_conf = self._safe_get_txn_conf()
1117
+
1118
+ if txn_conf.in_txn[-1] and self._txn_support_:
1119
+ txn_conf.qls[-1].extend(qls_with_args)
1120
+ return
1121
+
1122
+ return await self._execute(qls_with_args)
1123
+
1124
+ @staticmethod
1125
+ def _valid_data(data, object_name, relation, structure):
1126
+ required_fields = set(map(
1127
+ lambda f: f.name,
1128
+ filter(lambda f: f.required, structure.fields.values())
1129
+ ))
1130
+ if missing_fields := (required_fields - set(data.columns)):
1131
+ raise RequiredFieldUnfilled(f'缺少必填字段: {missing_fields}')
1132
+
1133
+ if not relation:
1134
+ return
1135
+
1136
+ for name, link_df in relation.items():
1137
+ if name not in structure.fields:
1138
+ continue
1139
+ field = structure.fields[name]
1140
+ if not field.is_multi_link:
1141
+ if name in data.columns:
1142
+ continue
1143
+
1144
+ raise SingleLinkInRelation(
1145
+ f'对象[{object_name}]的Link:[{name}]非multi link, '
1146
+ f'请直接作为入参data的{name}列提供, '
1147
+ f'值为对象{structure.fields[name].type}的业务主键'
1148
+ )
1149
+
1150
+ if not {'source', 'target'}.issubset(link_df.columns):
1151
+ raise ValueError("关联表必须包含source和target列")
1152
+
1153
+ @staticmethod
1154
+ def _valid_upsert(
1155
+ obj: ObjectTypeFrame,
1156
+ field_names: Iterable[str],
1157
+ bkey: str,
1158
+ exclusive_fields: List[str] = None,
1159
+ update_fields: List[str] = None
1160
+ ):
1161
+ if (
1162
+ update_fields
1163
+ and (missing := (set(update_fields) - set(field_names)))
1164
+ ):
1165
+ raise ValueError(f"update fields: {missing} 不在提供的数据中")
1166
+
1167
+ if exclusive_fields:
1168
+ if missing := (set(exclusive_fields) - set(field_names)):
1169
+ raise ValueError(f"exclusive fields: {missing} 不在提供的数据中")
1170
+
1171
+ valid_exclusive = [*obj.exclusive_fields, bkey]
1172
+ exclusive = (
1173
+ exclusive_fields[0]
1174
+ if len(exclusive_fields) == 1 else set(exclusive_fields)
1175
+ )
1176
+ if exclusive not in valid_exclusive:
1177
+ raise ValueError(f"exclusive fields: {exclusive_fields} 没有相应唯一约束")
1178
+
1179
+ @txn_support
1180
+ async def insert_df(
1181
+ self,
1182
+ object_name: str,
1183
+ data: pd.DataFrame,
1184
+ relation: Dict[str, pd.DataFrame] = None,
1185
+ chunksize: int = 500,
1186
+ enable_upsert: bool = False,
1187
+ update_fields: Iterable[str] = None,
1188
+ exclusive_fields: Iterable[str] = None,
1189
+ ) -> None:
1190
+ """以事务执行基于DataFrame字段信息的批量插入数据
1191
+
1192
+ Args:
1193
+ object_name: 被插入数据的对象名,需属于当前应用
1194
+ data: 要插入的数据,若有single link property,
1195
+ 则以列名为link_name@link_property_name的形式提供
1196
+ relation: 如有multi link,提供该字典用于补充link target信息,
1197
+ 键为link字段名,值为映射关系的DataFrame
1198
+ DataFrame中的source列需为插入对象的业务主键,
1199
+ target列需为link target的业务主键,
1200
+ 若有link property,则以property名为列名,提供在除source和target的列中
1201
+ chunksize: 单次最大行数
1202
+ enable_upsert: 是否组织成upsert句式
1203
+ update_fields: upsert句式下update的update fields列表,
1204
+ 涉及的fields需出现在data或relation中,
1205
+ 默认为提供的data列中除业务主键以外的fields
1206
+ exclusive_fields: upsert句式下update的exclusive fields列表,
1207
+ 涉及的fields需出现在data或relation中,
1208
+ 默认为业务主键
1209
+
1210
+ Notes:
1211
+
1212
+ 由于批量insert实现方式为组织 for-union clause 的 insert 语句,
1213
+ 而在其中查询self link只能查到已有数据,
1214
+ 无法查到 for-union clause 之前循环插入的结果,self link字段的数据将被单独抽出,
1215
+ 在 insert 后再用 for-union clause 的 update 语句更新
1216
+
1217
+
1218
+ .. admonition:: 示例(不涉及multi link)
1219
+
1220
+ .. code-block:: python
1221
+
1222
+ import pandas as pd
1223
+ from datetime import datetime
1224
+
1225
+ dm = DeepModel()
1226
+
1227
+ data = pd.DataFrame(
1228
+ {
1229
+ 'p_bool': [True, False],
1230
+ 'p_str': ['Hello', 'World'],
1231
+ 'p_local_datetime': [
1232
+ datetime(2021, 1, 1, 0, 0, 0),
1233
+ datetime(2021, 2, 1, 0, 0, 0),
1234
+ ],
1235
+ }
1236
+ )
1237
+ # 将data插入Demo对象
1238
+ dm.insert_df('Demo', data)
1239
+
1240
+ .. admonition:: 示例(涉及multi link)
1241
+
1242
+ .. code-block:: python
1243
+
1244
+ import pandas as pd
1245
+
1246
+ dm = DeepModel()
1247
+ # 对象主数据
1248
+ data = pd.DataFrame(
1249
+ {
1250
+ 'name': ['Alice', 'Bob', 'Carol']
1251
+ }
1252
+ )
1253
+ # 主数据的multi link字段target信息
1254
+ relation = {
1255
+ 'deck': pd.DataFrame(
1256
+ {
1257
+ # 一对多可用多行source与target的关联表示
1258
+ 'source': ['Alice', 'Alice', 'Bob', 'Carol'],
1259
+ 'target': [
1260
+ 'Dragon',
1261
+ 'Golem',
1262
+ 'Golem',
1263
+ 'Imp'
1264
+ ]
1265
+ }
1266
+ ),
1267
+ 'awards': pd.DataFrame(
1268
+ {
1269
+ 'source': ['Alice', 'Bob', 'Carol'],
1270
+ 'target': ['1st', '2nd', '3rd'],
1271
+ }
1272
+ )
1273
+ }
1274
+ dm.insert_df('User', data, relation=relation)
1275
+
1276
+ """
1277
+ if data.empty:
1278
+ logger.info("data为空,无DML执行")
1279
+ return
1280
+
1281
+ if object_name in self.objects:
1282
+ obj = self.objects[object_name]
1283
+ else:
1284
+ raise ObjectNotExist(
1285
+ f'DeepModel对象[{object_name}]在当前应用不存在,无法插入数据'
1286
+ )
1287
+ if obj.external:
1288
+ raise ExternalObjectReadOnly('外部对象只可读')
1289
+
1290
+ structure = ObjectStructure(name=obj.name, structure=obj.fields.values())
1291
+
1292
+ self._valid_data(data, object_name, relation, structure)
1293
+
1294
+ relation = relation or {}
1295
+ bkey = await self._get_bkey(obj)
1296
+ # data拼接relation df
1297
+ data = self._merge_relation(data, relation, structure, bkey)
1298
+ # 从data中分离出self-link更新信息
1299
+ data, self_link_dfs = self._split_self_link(data, relation, structure, bkey)
1300
+
1301
+ field_info = []
1302
+ tgt_main_field = {}
1303
+ # 准备bulk insert所需field信息
1304
+ for field in structure.fields.values():
1305
+ if field.name not in data.columns:
1306
+ continue
1307
+
1308
+ field_info.append(field)
1309
+
1310
+ if not field.is_link:
1311
+ continue
1312
+
1313
+ is_multi = field.is_multi_link
1314
+ name = field.name
1315
+ # 链接至其他对象,记录目标对象信息
1316
+ if is_multi:
1317
+ link_props = set(relation[name].columns).intersection(field.props)
1318
+ else:
1319
+ link_props = set(
1320
+ c[len(f'{name}@')::]
1321
+ for c in data.columns if c.startswith(f'{name}@')
1322
+ ).intersection(field.props)
1323
+ tgt_bkey = await self._get_bkey(field.target, object_name, name)
1324
+ tgt_main_field[name] = MainField(tgt_bkey, is_multi, link_props)
1325
+
1326
+ field_names = set(map(lambda f: f.name, field_info))
1327
+ if enable_upsert:
1328
+ self._valid_upsert(obj, field_names, bkey, exclusive_fields, update_fields)
1329
+
1330
+ exclusive_fields = set(exclusive_fields or {bkey}) & set(field_names)
1331
+ update_fields = set(update_fields or (field_names- {bkey})) & set(field_names)
1332
+ if enable_upsert and update_fields:
1333
+ insert_ql = bulk_upsert_by_fields(
1334
+ object_name, field_info, tgt_main_field,
1335
+ exclusive_fields, update_fields
1336
+ )
1337
+ else:
1338
+ insert_ql = bulk_insert_by_fields(object_name, field_info, tgt_main_field)
1339
+
1340
+ qls = []
1341
+ self._collect_qls(structure.fit(data), insert_ql, chunksize, qls)
1342
+ if self_link_dfs:
1343
+ for update_field, (update_df, main_field) in self_link_dfs.items():
1344
+ field = structure.fields[update_field]
1345
+ update_ql = bulk_update_by_fields(
1346
+ object_name, bkey, [field],
1347
+ {update_field: main_field}, [update_field]
1348
+ )
1349
+ self._collect_qls(update_df, update_ql, chunksize, qls)
1350
+
1351
+ await self.execute(qls)
1352
+
1353
+ def _collect_qls(
1354
+ self,
1355
+ data: pd.DataFrame,
1356
+ ql: str,
1357
+ chunksize: int,
1358
+ qls: List[QueryWithArgs]
1359
+ ):
1360
+ for i in range(0, len(data), chunksize):
1361
+ part = data.iloc[i: i + chunksize]
1362
+ kw_name = self.alias.get(BATCH_INSERT_KW)
1363
+ qls.append(QueryWithArgs(
1364
+ commands=ql.replace(
1365
+ f'${BATCH_INSERT_KW}', f'${kw_name}'
1366
+ ),
1367
+ kwargs={kw_name: part.to_json(orient='records', double_precision=15)}
1368
+ ))
1369
+
1370
+ async def get_object(
1371
+ self,
1372
+ object_name: str,
1373
+ raw: bool = True
1374
+ ) -> Optional[Union[ObjectTypeFrame, edgedb.Object]]:
1375
+ """获取指定对象名的对象
1376
+
1377
+ Args:
1378
+ object_name: 对象名,如对象名未提供module name部分,则默认module name为当前app module
1379
+ 如需查询其他module的对象,应以 module_name::object_name 的形式提供
1380
+ raw: 是否获取edgedb.Object数据,为False时将转换为ObjectTypeFrame数据
1381
+ 默认为True
1382
+
1383
+ """
1384
+ if '::' not in object_name:
1385
+ object_name = f'{self.appmodule}::{object_name}'
1386
+ objs = await AsyncDeepModel.query_object(
1387
+ self,
1388
+ f"{OBJECT_QUERY} filter .name = '{object_name}'",
1389
+ )
1390
+ if len(objs) == 0:
1391
+ return
1392
+
1393
+ if raw:
1394
+ return objs[0]
1395
+
1396
+ return format_obj(objs[0])
1397
+
1398
+ async def _get_bkey(
1399
+ self,
1400
+ obj: Union[ObjectTypeFrame, TargetField],
1401
+ source: str = None,
1402
+ name: str = None
1403
+ ) -> str:
1404
+ # 如可在object结构的annotations中取业务主键,则优先取,否则走接口
1405
+ if obj.info and BUSINESS_KEY in obj.info:
1406
+ return obj.info[BUSINESS_KEY]
1407
+ elif (code := obj.normalized_name) in self.model_objects:
1408
+ return self.model_objects[code].businessKey
1409
+
1410
+ assert isinstance(obj, TargetField)
1411
+ # Link 至非本应用对象,需单独查询
1412
+ tgt = ObjectElement.construct_from(self.model_objects[source]).links[name]
1413
+ tgt_model_info = await self.async_api.object.info(
1414
+ app=tgt.targetApp, object_code=tgt.targetObjectCode
1415
+ )
1416
+ return tgt_model_info.businessKey
1417
+
1418
+ @staticmethod
1419
+ def _split_self_link(data, relation, structure, bkey):
1420
+ self_link_dfs = {}
1421
+ for name in structure.self_link_fields:
1422
+ field = structure.fields[name]
1423
+ if (link_df := relation.get(name)) is not None:
1424
+ link_props = set(link_df.columns).intersection(field.props)
1425
+ self_link_dfs[name] = (
1426
+ structure.fit(data[[bkey, name]]),
1427
+ MainField(bkey, field.is_multi_link, link_props)
1428
+ )
1429
+ data = data.drop(columns=[name])
1430
+ elif name in data.columns:
1431
+ link_prop_cols = []
1432
+ link_props = []
1433
+
1434
+ for col in data.columns:
1435
+ if (
1436
+ col.startswith(f'{name}@')
1437
+ and ((prop_name := col[len(f'{name}@')::]) in field.props)
1438
+ ):
1439
+ link_prop_cols.append(col)
1440
+ link_props.append(prop_name)
1441
+
1442
+ self_link_dfs[name] = (
1443
+ structure.fit(data[[bkey, name, *link_prop_cols]]),
1444
+ MainField(bkey, field.is_multi_link, link_props)
1445
+ )
1446
+ data = data.drop(columns=[name, *link_prop_cols])
1447
+ return data, self_link_dfs
1448
+
1449
+ @staticmethod
1450
+ def _merge_relation(data, relation, structure, bkey):
1451
+ for name, link_df in relation.items():
1452
+ if name not in structure.fields:
1453
+ continue
1454
+ field = structure.fields[name]
1455
+ valid_cols = list({'source', 'target', *field.props} & set(link_df.columns))
1456
+ link_df = link_df[valid_cols]
1457
+ # for fit only
1458
+ temp_structure = ObjectStructure(
1459
+ field.type,
1460
+ [
1461
+ PtrInfo(name='source', target=TargetField(name='std::str')),
1462
+ PtrInfo(name='target', target=TargetField(name='std::str')),
1463
+ *[PtrInfo(**prop.dict()) for prop in field.properties]
1464
+ ]
1465
+ )
1466
+ link_df = temp_structure.fit(link_df)
1467
+ link = link_df.groupby('source').apply(_format_link, link_name=name)
1468
+ data = data.drop(columns=[name], errors='ignore')
1469
+ data = data.join(link.to_frame(name), on=bkey)
1470
+ return data
1471
+
1472
+ async def insert_df_pg(
1473
+ self,
1474
+ object_name: str,
1475
+ data: pd.DataFrame,
1476
+ chunk_size: int = 500,
1477
+ enable_upsert: bool = False,
1478
+ exclusive_fields: Iterable[str] = None,
1479
+ update_fields: Iterable[str] = None,
1480
+ ) -> None:
1481
+ """以事务直连pg执行基于DataFrame字段信息的批量插入数据
1482
+
1483
+ Args:
1484
+ object_name: 被插入数据的对象名,需属于当前应用
1485
+ data: 要插入的数据,只支持single property/link,且single link不可包含link property
1486
+ chunk_size: 单次最大行数
1487
+ enable_upsert: 是否组织成insert-on conflict do update 句式
1488
+ exclusive_fields: enable_upsert为True时的conflict字段,默认为business key
1489
+ update_fields: enable_upsert为True时的update字段,默认为除business key外所有已提供的字段
1490
+
1491
+ """
1492
+ if self.pg_dsn is None:
1493
+ raise ValueError('pg_dsn未提供')
1494
+
1495
+ if data.empty:
1496
+ return
1497
+
1498
+ if object_name in self.objects:
1499
+ obj = self.objects[object_name]
1500
+ else:
1501
+ raise ObjectNotExist(
1502
+ f'DeepModel对象[{object_name}]在当前应用不存在,无法插入数据'
1503
+ )
1504
+ if obj.external:
1505
+ raise ExternalObjectReadOnly('外部对象只可读')
1506
+
1507
+ structure = ObjectStructure(
1508
+ name=obj.name,
1509
+ structure=obj.fields.values(),
1510
+ include_id=True
1511
+ )
1512
+ self._valid_data(data, object_name, {}, structure)
1513
+
1514
+ field_info = []
1515
+ tgt_info = {}
1516
+ field_map = structure.fields
1517
+ bkey = await self._get_bkey(obj)
1518
+
1519
+ data = data.drop(columns=['id'], errors='ignore')
1520
+ data = data.assign(id=[uuid.uuid4() for _ in data.index], index=data.index)
1521
+
1522
+ for name, field in field_map.items():
1523
+ if name not in data.columns:
1524
+ continue
1525
+ if '@' in name:
1526
+ raise NotImplementedError('不支持插入包含link property的数据')
1527
+ if field.props:
1528
+ raise NotImplementedError('不支持插入有link property的link数据')
1529
+
1530
+ field_info.append(field)
1531
+ if not field.is_link:
1532
+ continue
1533
+
1534
+ if field.is_multi:
1535
+ raise NotImplementedError('不支持插入包含multi link/property的数据')
1536
+
1537
+ tgt = field.target
1538
+ tgt_bkey = await self._get_bkey(field.target, object_name, name)
1539
+ tgt_bkey_col = str(tgt.props[tgt_bkey].id)
1540
+ if tgt_col := tgt.props.get(field.target_col):
1541
+ tgt_info[name] = MainPgCol(
1542
+ target_bkey_col=tgt_bkey_col,
1543
+ target_col=str(tgt_col.id),
1544
+ )
1545
+ else:
1546
+ tgt_info[name] = MainPgCol(target_bkey_col=tgt_bkey_col, target_col='id')
1547
+
1548
+ # field_names will be used for data col order
1549
+ # so here use field_set as the set for field_names
1550
+ field_names = list(map(lambda f: f.name, field_info))
1551
+ field_set = set(field_names)
1552
+
1553
+ if enable_upsert:
1554
+ self._valid_upsert(obj, field_set, bkey, exclusive_fields, update_fields)
1555
+
1556
+ exclusive_fields = set(exclusive_fields or {bkey}) & field_set
1557
+ update_fields = (
1558
+ set(update_fields or (field_set - {bkey})) & field_set
1559
+ ) - {'id'}
1560
+
1561
+ update_fields = list(map(lambda uf: field_map[uf], update_fields))
1562
+ exclusive_fields = list(map(lambda ex: field_map[ex], exclusive_fields))
1563
+
1564
+ if enable_upsert:
1565
+ main_sql = batch_insert_pg(
1566
+ str(obj.id), field_info, tgt_info,
1567
+ exclusive_fields, update_fields
1568
+ )
1569
+ else:
1570
+ main_sql = batch_insert_pg(str(obj.id), field_info, tgt_info)
1571
+
1572
+ data = structure.fit(data, raw_pg=True)
1573
+ data = data.fillna(np.nan)
1574
+ data = data.replace([np.nan], [None])
1575
+ conn = await pg_conn(dsn=self.pg_dsn)
1576
+ try:
1577
+ async with conn.transaction():
1578
+ logger.debug(f"Prepare SQL: [{main_sql}]")
1579
+ main_sql_st = await conn.prepare(main_sql)
1580
+ data = data[field_names]
1581
+ for i in range(0, len(data), chunk_size):
1582
+ part = data.iloc[i: i + chunk_size]
1583
+ await main_sql_st.executemany(part.itertuples(index=False, name=None))
1584
+ finally:
1585
+ await conn.close()
1586
+
1587
+ @asynccontextmanager
1588
+ async def start_transaction(self, flatten: bool = False):
1589
+ """开启事务
1590
+
1591
+ 上下文管理器,使用with语法开启上下文,上下文中的ql将作为事务执行
1592
+ 退出with语句块后,事务将立即执行,执行过程中如果报错会直接抛出
1593
+
1594
+ .. admonition:: 示例
1595
+
1596
+ .. code-block:: python
1597
+
1598
+ import pandas as pd
1599
+
1600
+ dm = DeepModel()
1601
+
1602
+ data = pd.DataFrame(
1603
+ {
1604
+ 'name': ['Alice', 'Bob', 'Carol'],
1605
+ 'deck': [
1606
+ "Dragon",
1607
+ "Golem",
1608
+ "Imp"
1609
+ ],
1610
+ 'awards': [
1611
+ "1st",
1612
+ "2nd",
1613
+ "3rd"
1614
+ ],
1615
+ }
1616
+ )
1617
+
1618
+ async with dm.start_transaction():
1619
+ await dm.execute("delete User")
1620
+ await dm.insert_df("User", data)
1621
+
1622
+
1623
+ Important:
1624
+
1625
+ 仅 :func:`insert_df` :func:`execute` 方法支持在事务中执行
1626
+
1627
+ """
1628
+ try:
1629
+ self._txn_.get()
1630
+ except LookupError:
1631
+ self._txn_.set(_TxnConfig())
1632
+
1633
+ bak_flatten = self._txn_.get().flatten
1634
+ self._txn_.get().in_txn.append(True)
1635
+
1636
+ if flatten and not self._txn_.get().flatten:
1637
+ force_submit = True
1638
+ else:
1639
+ force_submit = False
1640
+
1641
+ self._txn_.get().flatten = bak_flatten or flatten
1642
+
1643
+ if not self._txn_.get().flatten:
1644
+ self._txn_.get().qls.append([])
1645
+
1646
+ try:
1647
+ yield
1648
+ if force_submit or not self._txn_.get().flatten:
1649
+ if qls := self._txn_.get().qls.pop():
1650
+ await self._execute(qls)
1651
+ finally:
1652
+ self._txn_.get().in_txn.pop()
1653
+ self._txn_.get().flatten = bak_flatten
1654
+
1655
+ @contextmanager
1656
+ def with_globals(self, globals_):
1657
+ if not self.direct_access:
1658
+ try:
1659
+ yield
1660
+ finally:
1661
+ raise NotImplemented('非直连模式不支持设置state信息')
1662
+ else:
1663
+ bak_cli = self.client
1664
+ try:
1665
+ self.client = self.client.with_globals(**globals_)
1666
+ yield
1667
+ finally:
1668
+ self.client = bak_cli
1669
+
1670
+ @contextmanager
1671
+ def without_globals(self, *global_names):
1672
+ if not self.direct_access:
1673
+ try:
1674
+ yield
1675
+ finally:
1676
+ raise NotImplemented('非直连模式不支持设置state信息')
1677
+ else:
1678
+ bak_cli = self.client
1679
+ try:
1680
+ self.client = self.client.without_globals(*global_names)
1681
+ yield
1682
+ finally:
1683
+ self.client = bak_cli
1684
+
1685
+
1686
+ class DeepModel(AsyncDeepModel, metaclass=SyncMeta):
1687
+ synchronize = (
1688
+ 'query_object', 'query', 'query_df',
1689
+ 'execute', 'get_object',
1690
+ 'insert_df', 'insert_df_pg'
1691
+ )
1692
+
1693
+ if TYPE_CHECKING: # pragma: no cover
1694
+ def query_object(self, ql: str, **kwargs) -> List[Any]:
1695
+ ...
1696
+
1697
+ def query(self, ql: str, **kwargs) -> List[Any]:
1698
+ ...
1699
+
1700
+ def query_df(self, ql: str, **kwargs) -> pd.DataFrame:
1701
+ ...
1702
+
1703
+ def execute(
1704
+ self,
1705
+ qls: Union[str, List[str], List[QueryWithArgs]],
1706
+ **kwargs
1707
+ ) -> Optional[List]:
1708
+ ...
1709
+
1710
+ def get_object(
1711
+ self,
1712
+ object_name: str,
1713
+ raw: bool = True
1714
+ ) -> Optional[Union[ObjectTypeFrame, edgedb.Object]]:
1715
+ ...
1716
+
1717
+ def insert_df(
1718
+ self,
1719
+ object_name: str,
1720
+ data: pd.DataFrame,
1721
+ relation: Dict[str, pd.DataFrame] = None,
1722
+ chunksize: int = 500,
1723
+ enable_upsert: bool = False,
1724
+ update_fields: Iterable[str] = None,
1725
+ exclusive_fields: Iterable[str] = None,
1726
+ ) -> None:
1727
+ ...
1728
+
1729
+ def insert_df_pg(
1730
+ self,
1731
+ object_name: str,
1732
+ data: pd.DataFrame,
1733
+ chunk_size: int = 500,
1734
+ enable_upsert: bool = False,
1735
+ exclusive_fields: List[str] = None,
1736
+ update_fields: List[str] = None,
1737
+ ) -> None:
1738
+ ...
1739
+
1740
+ @contextmanager
1741
+ def start_transaction(self, flatten: bool = False):
1742
+ """开启事务
1743
+
1744
+ 上下文管理器,使用with语法开启上下文,上下文中的ql将作为事务执行
1745
+ 退出with语句块后,事务将立即执行,执行过程中如果报错会直接抛出
1746
+
1747
+ .. admonition:: 示例
1748
+
1749
+ .. code-block:: python
1750
+
1751
+ import pandas as pd
1752
+
1753
+ dm = DeepModel()
1754
+
1755
+ data = pd.DataFrame(
1756
+ {
1757
+ 'name': ['Alice', 'Bob', 'Carol'],
1758
+ 'deck': [
1759
+ "Dragon",
1760
+ "Golem",
1761
+ "Imp"
1762
+ ],
1763
+ 'awards': [
1764
+ "1st",
1765
+ "2nd",
1766
+ "3rd"
1767
+ ],
1768
+ }
1769
+ )
1770
+
1771
+ with dm.start_transaction():
1772
+ dm.execute("delete User")
1773
+ dm.insert_df("User", data)
1774
+
1775
+
1776
+ Important:
1777
+
1778
+ 仅 :func:`insert_df` :func:`execute` 方法支持在事务中执行
1779
+
1780
+ """
1781
+ try:
1782
+ self._txn_.get()
1783
+ except LookupError:
1784
+ self._txn_.set(_TxnConfig())
1785
+
1786
+ bak_flatten = self._txn_.get().flatten
1787
+ self._txn_.get().in_txn.append(True)
1788
+
1789
+ if flatten and not self._txn_.get().flatten:
1790
+ force_submit = True
1791
+ else:
1792
+ force_submit = False
1793
+
1794
+ self._txn_.get().flatten = bak_flatten or flatten
1795
+
1796
+ if not self._txn_.get().flatten:
1797
+ self._txn_.get().qls.append([])
1798
+
1799
+ try:
1800
+ yield
1801
+ if force_submit or not self._txn_.get().flatten:
1802
+ if qls := self._txn_.get().qls.pop():
1803
+ evloop.run(self._execute(qls))
1804
+ finally:
1805
+ self._txn_.get().in_txn.pop()
1806
+ self._txn_.get().flatten = bak_flatten