deepfos 1.1.4__tar.gz → 1.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. deepfos-1.1.6/MANIFEST.in +5 -0
  2. {deepfos-1.1.4 → deepfos-1.1.6}/PKG-INFO +1 -1
  3. {deepfos-1.1.4 → deepfos-1.1.6}/README.md +1 -0
  4. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/base.py +24 -9
  5. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/_base.py +2 -2
  6. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/logictable/tablemodel.py +7 -19
  7. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/dbkits.py +36 -31
  8. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/apvlprocess.py +14 -7
  9. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/bizmodel.py +31 -24
  10. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/datatable.py +106 -33
  11. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/dimension.py +6 -19
  12. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/finmodel.py +81 -65
  13. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/journal_template.py +19 -64
  14. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/pyscript.py +1 -1
  15. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/smartlist.py +2 -2
  16. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/exceptions/hook.py +17 -17
  17. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lazy.py +4 -1
  18. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/asynchronous.py +68 -43
  19. deepfos-1.1.6/deepfos/lib/discovery.py +206 -0
  20. deepfos-1.1.6/deepfos/lib/eureka.py +138 -0
  21. deepfos-1.1.6/deepfos/lib/nacos.py +225 -0
  22. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/options.py +111 -28
  23. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/translation.py +24 -12
  24. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos.egg-info/PKG-INFO +1 -1
  25. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos.egg-info/SOURCES.txt +2 -0
  26. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos.egg-info/requires.txt +2 -1
  27. {deepfos-1.1.4 → deepfos-1.1.6}/setup.cfg +2 -1
  28. {deepfos-1.1.4 → deepfos-1.1.6}/setup.py +1 -1
  29. deepfos-1.1.4/MANIFEST.in +0 -4
  30. deepfos-1.1.4/deepfos/lib/eureka.py +0 -94
  31. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/__init__.py +0 -0
  32. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/_version.py +0 -0
  33. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/algo/__init__.py +0 -0
  34. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/algo/graph.py +0 -0
  35. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_1/__init__.py +0 -0
  36. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_1/business_model.py +0 -0
  37. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_1/dimension.py +0 -0
  38. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_1/models/__init__.py +0 -0
  39. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_1/models/business_model.py +0 -0
  40. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_1/models/dimension.py +0 -0
  41. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_2/__init__.py +0 -0
  42. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_2/dimension.py +0 -0
  43. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_2/models/__init__.py +0 -0
  44. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/V1_2/models/dimension.py +0 -0
  45. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/__init__.py +0 -0
  46. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/account.py +0 -0
  47. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/accounting_engines.py +0 -0
  48. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/app.py +0 -0
  49. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/approval_process.py +0 -0
  50. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/business_model.py +0 -0
  51. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/consolidation.py +0 -0
  52. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/datatable.py +0 -0
  53. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/deepfos_task.py +0 -0
  54. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/dimension.py +0 -0
  55. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/financial_model.py +0 -0
  56. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/journal_template.py +0 -0
  57. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/memory_financial_model.py +0 -0
  58. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/__init__.py +0 -0
  59. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/account.py +0 -0
  60. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/accounting_engines.py +0 -0
  61. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/app.py +0 -0
  62. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/approval_process.py +0 -0
  63. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/base.py +0 -0
  64. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/business_model.py +0 -0
  65. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/consolidation.py +0 -0
  66. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/datatable_mysql.py +0 -0
  67. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/deepfos_task.py +0 -0
  68. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/dimension.py +0 -0
  69. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/financial_model.py +0 -0
  70. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/journal_template.py +0 -0
  71. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/memory_financial_model.py +0 -0
  72. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/platform.py +0 -0
  73. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/python.py +0 -0
  74. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/reconciliation_engine.py +0 -0
  75. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/reconciliation_report.py +0 -0
  76. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/role_strategy.py +0 -0
  77. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/smartlist.py +0 -0
  78. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/space.py +0 -0
  79. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/system.py +0 -0
  80. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/variable.py +0 -0
  81. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/models/workflow.py +0 -0
  82. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/platform.py +0 -0
  83. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/python.py +0 -0
  84. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/reconciliation_engine.py +0 -0
  85. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/reconciliation_report.py +0 -0
  86. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/role_strategy.py +0 -0
  87. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/smartlist.py +0 -0
  88. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/space.py +0 -0
  89. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/system.py +0 -0
  90. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/variable.py +0 -0
  91. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/api/workflow.py +0 -0
  92. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/boost/__init__.py +0 -0
  93. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/boost/jstream.c +0 -0
  94. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/boost/jstream.pyx +0 -0
  95. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/boost/pandas.c +0 -0
  96. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/boost/pandas.pyx +0 -0
  97. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/boost/py_jstream.py +0 -0
  98. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/boost/py_pandas.py +0 -0
  99. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/cache.py +0 -0
  100. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/config.py +0 -0
  101. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/__init__.py +0 -0
  102. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/__init__.py +0 -0
  103. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/constants.py +0 -0
  104. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/cube.py +0 -0
  105. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/formula.py +0 -0
  106. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/syscube.py +0 -0
  107. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/typing.py +0 -0
  108. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/cube/utils.py +0 -0
  109. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/__init__.py +0 -0
  110. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/_base.py +0 -0
  111. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/dimcreator.py +0 -0
  112. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/dimension.py +0 -0
  113. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/dimexpr.py +0 -0
  114. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/dimmember.py +0 -0
  115. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/eledimension.py +0 -0
  116. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/filters.py +0 -0
  117. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/dimension/sysdimension.py +0 -0
  118. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/logictable/__init__.py +0 -0
  119. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/logictable/_cache.py +0 -0
  120. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/logictable/_operator.py +0 -0
  121. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/logictable/nodemixin.py +0 -0
  122. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/core/logictable/sqlcondition.py +0 -0
  123. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/__init__.py +0 -0
  124. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/cipher.py +0 -0
  125. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/clickhouse.py +0 -0
  126. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/connector.py +0 -0
  127. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/daclickhouse.py +0 -0
  128. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/dameng.py +0 -0
  129. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/damysql.py +0 -0
  130. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/deepengine.py +0 -0
  131. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/gauss.py +0 -0
  132. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/kingbase.py +0 -0
  133. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/mysql.py +0 -0
  134. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/oracle.py +0 -0
  135. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/postgresql.py +0 -0
  136. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/sqlserver.py +0 -0
  137. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/db/utils.py +0 -0
  138. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/__init__.py +0 -0
  139. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/accounting.py +0 -0
  140. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/base.py +0 -0
  141. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/fact_table.py +0 -0
  142. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/reconciliation.py +0 -0
  143. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/rolestrategy.py +0 -0
  144. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/variable.py +0 -0
  145. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/element/workflow.py +0 -0
  146. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/exceptions/__init__.py +0 -0
  147. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/__init__.py +0 -0
  148. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/_javaobj.py +0 -0
  149. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/concurrency.py +0 -0
  150. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/constant.py +0 -0
  151. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/decorator.py +0 -0
  152. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/deepchart.py +0 -0
  153. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/deepux.py +0 -0
  154. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/filterparser.py +0 -0
  155. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/httpcli.py +0 -0
  156. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/jsonstreamer.py +0 -0
  157. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/patch.py +0 -0
  158. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/redis.py +0 -0
  159. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/stopwatch.py +0 -0
  160. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/subtask.py +0 -0
  161. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/sysutils.py +0 -0
  162. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/lib/utils.py +0 -0
  163. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos/local.py +0 -0
  164. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos.egg-info/dependency_links.txt +0 -0
  165. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos.egg-info/not-zip-safe +0 -0
  166. {deepfos-1.1.4 → deepfos-1.1.6}/deepfos.egg-info/top_level.txt +0 -0
  167. {deepfos-1.1.4 → deepfos-1.1.6}/tests/__init__.py +0 -0
  168. {deepfos-1.1.4 → deepfos-1.1.6}/versioneer.py +0 -0
@@ -0,0 +1,5 @@
1
+ include versioneer.py
2
+ include deepfos/_version.py
3
+ include *.pyi
4
+ recursive-include deepfos/boost *.pyx
5
+ recursive-include deepfos/boost *.c
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: deepfos
3
- Version: 1.1.4
3
+ Version: 1.1.6
4
4
  Summary: Collecions of useful and handy tools for deepfos platform
5
5
  Home-page: http://py.deepfos.com
6
6
  Author: deepfos-python-team
@@ -34,4 +34,5 @@ DeepFOS SDK for Python
34
34
  [@陈熙](https://e.gitee.com/proinnova/members/trend/axisarsae)
35
35
  [@陈思聪](https://e.gitee.com/proinnova/members/trend/proinnova_sicong_chen)
36
36
  [@杨璟](https://e.gitee.com/proinnova/members/trend/jyang-0506)
37
+ [@李扬](https://e.gitee.com/proinnova/members/trend/ryan_li1384)
37
38
  [@陈文雍](https://e.gitee.com/proinnova/members/trend/calvinstk)
@@ -21,7 +21,7 @@ from deepfos.cache import Manager, AppSeperatedLRUCache
21
21
  from deepfos.lib.httpcli import AioHttpCli
22
22
  from deepfos.lib.utils import concat_url, retry, to_version_tuple, repr_version
23
23
  from deepfos.lib.asynchronous import evloop
24
- from deepfos.lib.eureka import Eureka
24
+ from deepfos.lib.discovery import ServiceDiscovery
25
25
  from deepfos.lib.constant import UNSET, RE_SYS_SERVER_PARSER
26
26
  from deepfos.options import OPTION
27
27
  from deepfos.exceptions import APIResponseError, APIRequestError
@@ -352,6 +352,11 @@ class Route:
352
352
  req.update_default(func(ins, *args, **kwargs))
353
353
  url, body, ext_header = req.url, req.body, req.header
354
354
  raw_result = model is None
355
+ has_req_callback = (
356
+ isinstance(ins.root, _DynamicAPIMixin)
357
+ and ins.root.on_success is not None
358
+ and ins.root.on_failure is not None
359
+ )
355
360
 
356
361
  if ext_header is None:
357
362
  header = ins.header
@@ -416,6 +421,9 @@ class Route:
416
421
  except OSError as e: # pragma: no cover
417
422
  if not OPTION.api.dump_always and OPTION.api.dump_on_failure:
418
423
  req.to_curl(header)
424
+
425
+ if has_req_callback:
426
+ ins.root.on_failure()
419
427
  raise APIRequestError(e) from None
420
428
 
421
429
  # -----------------------------------------------------------------------------
@@ -433,7 +441,12 @@ class Route:
433
441
  )
434
442
  flag, obj, err = False, None, f"[code: {status_code}] ErrMsg from server: {text}"
435
443
 
444
+ if has_req_callback:
445
+ ins.root.on_failure()
446
+
436
447
  else:
448
+ if has_req_callback:
449
+ ins.root.on_success()
437
450
  try:
438
451
  resp = json.loads(text)
439
452
  if 'status' not in resp:
@@ -572,6 +585,8 @@ class _DynamicAPIMixin:
572
585
  module_type: str = UNSET
573
586
  server_known = False
574
587
  version = None
588
+ on_success = None
589
+ on_failure = None
575
590
 
576
591
  def get_module_id(self, version: Union[float, str], module_id: str):
577
592
  if (module_type := self.module_type) is UNSET:
@@ -626,10 +641,10 @@ class DynamicAPIBase(SyncAPIBase, _DynamicAPIMixin):
626
641
  return self._add_to_memo(server_meta)
627
642
 
628
643
  def set_url(self, server_name):
629
- if OPTION.general.use_eureka: # pragma: no cover
630
- system_env = OPTION.general.task_info.get('system_env') or ''
631
- server_name = f"{system_env}{server_name}"
632
- self.base_url = concat_url(Eureka.sync_get_url(server_name), self.endpoint)
644
+ if OPTION.discovery.enabled: # pragma: no cover
645
+ discovery = ServiceDiscovery.instantiate()
646
+ self.on_success, self.on_failure, base_url = discovery.sync_get_url(server_name, include_cb=True)
647
+ self.base_url = concat_url(base_url, self.endpoint)
633
648
  self.server_known = True
634
649
  else:
635
650
  super().set_url(server_name)
@@ -669,10 +684,10 @@ class ADynamicAPIBase(AsyncAPIBase, _DynamicAPIMixin):
669
684
  return self
670
685
 
671
686
  async def set_url(self, server_name):
672
- if OPTION.general.use_eureka: # pragma: no cover
673
- system_env = OPTION.general.task_info.get('system_env') or ''
674
- server_name = f"{system_env}{server_name}"
675
- self.base_url = concat_url(await Eureka.get_url(server_name), self.endpoint)
687
+ if OPTION.discovery.enabled: # pragma: no cover
688
+ discovery = ServiceDiscovery.instantiate()
689
+ self.on_success, self.on_failure, base_url = await discovery.get_url(server_name, include_cb=True)
690
+ self.base_url = concat_url(base_url, self.endpoint)
676
691
  self.server_known = True
677
692
  else:
678
693
  super().set_url(server_name)
@@ -1,6 +1,6 @@
1
1
  from collections import deque
2
2
  from contextlib import contextmanager
3
- from typing import Dict, List, Sequence, Optional, Union, Deque
3
+ from typing import Dict, List, Sequence, Optional, Union, Deque, ContextManager
4
4
 
5
5
  import numpy as np
6
6
  import pandas as pd
@@ -299,7 +299,7 @@ class CubeBase:
299
299
  self,
300
300
  general_fix: str = None,
301
301
  on_dim: str = None,
302
- ) -> CubeFixer:
302
+ ) -> ContextManager[CubeFixer]:
303
303
  """
304
304
  执行成员公式
305
305
 
@@ -7,12 +7,7 @@ from ._operator import OpCombineError
7
7
  from loguru import logger
8
8
  from deepfos.lib.decorator import cached_property
9
9
  from deepfos.element.datatable import (
10
- DataTableMySQL, DataTableClickHouse,
11
- DataTableOracle, DataTableSQLServer, DataTableDeepEngine, DataTablePostgreSQL, DataTableDaMeng,
12
- DataTableGauss, DataTableKingBase,
13
- AsyncDataTableMySQL, AsyncDataTableClickHouse, AsyncDataTableOracle, AsyncDataTableSQLServer,
14
- AsyncDataTableKingBase, AsyncDataTableGauss, AsyncDataTableDaMeng, AsyncDataTablePostgreSQL,
15
- AsyncDataTableDeepEngine, get_table_class
10
+ get_table_class, T_DatatableInstance, T_AsyncDatatableInstance
16
11
  )
17
12
  from deepfos.api.models.app import ConfirmElementInfoDto
18
13
  import pandas as pd
@@ -23,16 +18,6 @@ from contextlib import contextmanager
23
18
  from collections import defaultdict
24
19
  import copy
25
20
 
26
- T_AbsDataTable = Union[
27
- DataTableMySQL, DataTableClickHouse, DataTableOracle, DataTableSQLServer,
28
- DataTableKingBase, DataTableGauss, DataTableDaMeng, DataTablePostgreSQL, DataTableDeepEngine
29
- ]
30
- T_AbsAsyncDataTable = Union[
31
- AsyncDataTableMySQL, AsyncDataTableClickHouse, AsyncDataTableOracle, AsyncDataTableSQLServer,
32
- AsyncDataTableKingBase, AsyncDataTableGauss, AsyncDataTableDaMeng, AsyncDataTablePostgreSQL,
33
- AsyncDataTableDeepEngine
34
- ]
35
-
36
21
 
37
22
  # -----------------------------------------------------------------------------
38
23
  # utils
@@ -41,7 +26,10 @@ class TableInfo(ConfirmElementInfoDto):
41
26
  serverName: Optional[str]
42
27
 
43
28
 
44
- def get_datatable(table_info: Union[TableInfo, dict], sync=True) -> Union[T_AbsDataTable, T_AbsAsyncDataTable]:
29
+ def get_datatable(
30
+ table_info: Union[TableInfo, dict],
31
+ sync=True
32
+ ) -> Union[T_DatatableInstance, T_AsyncDatatableInstance]:
45
33
  if isinstance(table_info, TableInfo):
46
34
  init_args = {
47
35
  'element_name': table_info.elementName,
@@ -196,7 +184,7 @@ class MetaTable(MetaNodeMixin):
196
184
  super().__init__(cls_name, supercls, attrdict)
197
185
 
198
186
  @cached_property
199
- def datatable(cls) -> T_AbsDataTable:
187
+ def datatable(cls) -> T_DatatableInstance:
200
188
  """数据表元素"""
201
189
  if cls.__datatable is not None:
202
190
  return cls.__datatable
@@ -207,7 +195,7 @@ class MetaTable(MetaNodeMixin):
207
195
  return tbl
208
196
 
209
197
  @cached_property
210
- def async_datatable(cls) -> T_AbsAsyncDataTable:
198
+ def async_datatable(cls) -> T_AsyncDatatableInstance:
211
199
  """异步数据表元素"""
212
200
  if cls.__async_datatable is not None:
213
201
  return cls.__async_datatable
@@ -1,6 +1,8 @@
1
1
  import re
2
2
  from typing import Dict, Union, List, Iterable, Tuple
3
3
  from typing_extensions import Protocol
4
+
5
+ from deepfos.cache import Manager
4
6
  from deepfos.lib.decorator import cached_property
5
7
 
6
8
  from cachetools import TTLCache
@@ -26,7 +28,6 @@ T_DataInfo = Dict[str, Union[Dict, DataTable, BaseElementInfo]]
26
28
 
27
29
  @singleton
28
30
  class APIFinder:
29
- @cache_async
30
31
  async def modules(self):
31
32
  mdl = await SpaceAPI(sync=False).module.get_usable_module()
32
33
  return mdl or []
@@ -56,9 +57,8 @@ class BaseSqlParser:
56
57
  api_cls = None
57
58
 
58
59
  def __init__(self):
59
- self.table_cache = TTLCache(maxsize=100, ttl=3600)
60
+ self.table_cache = Manager.create_cache(TTLCache, maxsize=100, ttl=3600)
60
61
 
61
- @cache_async
62
62
  async def build_api(self):
63
63
  return await APIFinder().find_api(self.api_cls, version=1.0)
64
64
 
@@ -244,8 +244,9 @@ class DataframeSQLConvertor:
244
244
  Returns:
245
245
  sql语句生成器
246
246
 
247
- See Also:
248
- :func:`df_to_sql`
247
+ Attention:
248
+ 当单个DataFrame生成的sql太长导致无法入库,可以指定 ``chuncksize`` ,
249
+ 使DataFrame生成多条sql。
249
250
 
250
251
  """
251
252
  # 获取sql
@@ -262,11 +263,36 @@ class DataframeSQLConvertor:
262
263
 
263
264
  @classmethod
264
265
  def _quote_escape(cls, value):
265
- if value is null:
266
- return null
266
+ if pd.isna(value):
267
+ return value
267
268
  if not isinstance(value, str):
268
269
  return str(value)
269
270
  return f"'{cls.escape_string(value)}'"
271
+
272
+ @staticmethod
273
+ def format_datetime(maybe_datetime: pd.Series) -> pd.Series:
274
+ return "'" + maybe_datetime.dt.strftime("%Y-%m-%d %H:%M:%S") + "'"
275
+
276
+ @classmethod
277
+ def format_series(cls, series: pd.Series) -> pd.Series:
278
+ """格式化Series以适合sql语句
279
+
280
+ 1. 日期型重置格式到秒级别
281
+ 2. 字符串型列转义、加引号
282
+
283
+ Args:
284
+ series (pd.Series): 需要格式化的Series
285
+
286
+ Returns:
287
+ pd.Series: 格式化后的新的Series,不影响原Series
288
+ """
289
+ # 对日期型重置格式到秒级别
290
+ if is_datetime64_dtype(series.dtype):
291
+ return cls.format_datetime(series)
292
+ # 对字符串型列转义,加引号
293
+ elif not is_numeric_dtype(series.dtype):
294
+ return series.apply(cls._quote_escape)
295
+ return series.copy()
270
296
 
271
297
  def convert(
272
298
  self,
@@ -288,41 +314,20 @@ class DataframeSQLConvertor:
288
314
  Returns:
289
315
  sql语句
290
316
 
291
- See Also:
292
- 如果单条sql语句太长导致无法入库,可以使用 :func:`gen_sql`,
293
- 指定 ``chuncksize`` ,该方法将把一条较大的sql拆分成多条执行。
294
-
295
317
  """
296
318
  if dataframe.empty:
297
319
  return ''
298
-
299
- data_df = dataframe.copy()
300
- # 获取日期型与非数字型的列
301
- datetime_col = [is_datetime64_dtype(x) for x in data_df.dtypes] # 获取日期型的列
302
- # 对日期型重置格式到秒级别
303
- data_df.loc[:, datetime_col] = data_df.loc[:, datetime_col] \
304
- .apply(lambda x: np.where(x.isna(), np.NAN, self.format_datetime(x)))
305
-
320
+ # 格式化Series以适合sql语句
321
+ data_df = dataframe.apply(self.format_series)
306
322
  # 空值填充
307
323
  data_df = data_df.fillna(null)
308
- # 获取非数字型的列
309
- str_like_cols = [
310
- (not (is_numeric_dtype(x) or is_datetime))
311
- for x, is_datetime in zip(data_df.dtypes, datetime_col)
312
- ]
313
- # 对字符串型列转义,加引号
314
- data_df.loc[:, str_like_cols] = data_df.loc[:, str_like_cols].applymap(self._quote_escape)
315
324
  # 全部转化为字符串类型
316
325
  data_df = data_df.astype(str, errors='ignore')
317
326
  values = "(" + pd.Series(data_df.values.tolist()).str.join(',') + ")"
318
327
  columns = self.build_column_string(dataframe.columns)
319
328
  return self.build_sql(columns, values, tablename, updatecol, **opts)
320
329
 
321
- @staticmethod
322
- def format_datetime(maybe_datetime):
323
- return "'" + maybe_datetime.dt.strftime("%Y-%m-%d %H:%M:%S") + "'"
324
-
325
- def build_column_string(self, columns):
330
+ def build_column_string(self, columns: pd.Index) -> str:
326
331
  if self.quote_char:
327
332
  columns = ','.join(columns.map(
328
333
  lambda x: f'{self.quote_char}{x}{self.quote_char}'
@@ -1,8 +1,8 @@
1
1
  from typing import List, Dict, Optional, Union, Tuple, TYPE_CHECKING, Any # noqa
2
2
 
3
3
  from .base import ElementBase, SyncMeta
4
- from .datatable import (DataTableMySQL, DataTableClickHouse, get_table_class, AsyncDataTableMySQL,
5
- AsyncDataTableClickHouse)
4
+ from .datatable import (DataTableMySQL, get_table_class, AsyncDataTableMySQL,
5
+ T_AsyncDatatableInstance, T_DatatableInstance)
6
6
  from deepfos.lib.decorator import cached_property
7
7
  from deepfos.lib.constant import UNSET
8
8
  from deepfos.api.approval_process import ApprovalProcessAPI
@@ -34,20 +34,27 @@ class AsyncApprovalProcess(ElementBase[ApprovalProcessAPI]):
34
34
  return meta
35
35
 
36
36
  @cached_property
37
- def record_table(self) -> Union[DataTableMySQL, DataTableClickHouse]:
37
+ def record_table(self) -> T_DatatableInstance:
38
38
  """审批记录表"""
39
39
  ctrl_info = self.meta.processInfo.controlInfo
40
- cls = get_table_class(ctrl_info.approvalRecordTableElementType)
40
+ if (ele_type := ctrl_info.approvalRecordTableElementType) is None:
41
+ cls = DataTableMySQL
42
+ else:
43
+ cls = get_table_class(ele_type)
41
44
  return cls(
42
45
  element_name=ctrl_info.approvalRecordTableName,
43
46
  folder_id=ctrl_info.approvalRecordTableFolderId,
44
- path=ctrl_info.approvalRecordTablePath)
47
+ path=ctrl_info.approvalRecordTablePath
48
+ )
45
49
 
46
50
  @cached_property
47
- def async_record_table(self) -> Union[AsyncDataTableMySQL, AsyncDataTableClickHouse]:
51
+ def async_record_table(self) -> T_AsyncDatatableInstance:
48
52
  """审批记录表"""
49
53
  ctrl_info = self.meta.processInfo.controlInfo
50
- cls = get_table_class(ctrl_info.approvalRecordTableElementType, sync=False)
54
+ if (ele_type := ctrl_info.approvalRecordTableElementType) is None:
55
+ cls = AsyncDataTableMySQL
56
+ else:
57
+ cls = get_table_class(ele_type, sync=False)
51
58
  return cls(
52
59
  element_name=ctrl_info.approvalRecordTableName,
53
60
  folder_id=ctrl_info.approvalRecordTableFolderId,
@@ -438,7 +438,7 @@ class AsyncBusinessModel(ElementBase[BusinessModelAPI]):
438
438
  primary_kv = {**primary_kv}
439
439
  return primary_kv
440
440
 
441
- def copy_rows(
441
+ async def copy_rows(
442
442
  self,
443
443
  config: Union[CopyConfig, Dict]
444
444
  ):
@@ -485,8 +485,8 @@ class AsyncBusinessModel(ElementBase[BusinessModelAPI]):
485
485
  config = CopyConfig().load_config(config)
486
486
 
487
487
  for table, conf in config.items():
488
- dt = self.logic_tables[table].datatable
489
- dt.copy_rows(**conf)
488
+ dt = self.logic_tables[table].async_datatable
489
+ await dt.copy_rows(**conf)
490
490
 
491
491
  @deprecated(
492
492
  replacement='set_approval_ex',
@@ -1183,6 +1183,7 @@ class BusinessModel(AsyncBusinessModel, metaclass=SyncMeta):
1183
1183
  'set_approval_ex',
1184
1184
  'save',
1185
1185
  'attach',
1186
+ 'copy_rows'
1186
1187
  )
1187
1188
 
1188
1189
  if TYPE_CHECKING: # pragma: no cover
@@ -1233,30 +1234,36 @@ class BusinessModel(AsyncBusinessModel, metaclass=SyncMeta):
1233
1234
  ...
1234
1235
 
1235
1236
  def save(
1236
- self,
1237
- data: Union[pd.DataFrame, Dict[str, pd.DataFrame]],
1238
- allow_detached_data: bool = True,
1239
- allow_detached_table: bool = False,
1240
- auto_format: bool = True,
1241
- check_db: bool = True,
1242
- check_logical: bool = True,
1243
- check_field: bool = True,
1244
- enable_pre_save: bool = True,
1245
- enable_post_save: bool = True,
1237
+ self,
1238
+ data: Union[pd.DataFrame, Dict[str, pd.DataFrame]],
1239
+ allow_detached_data: bool = True,
1240
+ allow_detached_table: bool = False,
1241
+ auto_format: bool = True,
1242
+ check_db: bool = True,
1243
+ check_logical: bool = True,
1244
+ check_field: bool = True,
1245
+ enable_pre_save: bool = True,
1246
+ enable_post_save: bool = True,
1246
1247
  ) -> List[ModelDataReturnDTO]:
1247
1248
  ...
1248
1249
 
1249
1250
  def attach(
1250
- self,
1251
- data: Union[pd.DataFrame, Dict[str, pd.DataFrame]],
1252
- primary_kv: Union[Any, Dict[str, Any]],
1253
- parent_kv: Union[Any, Dict[str, Any]] = None,
1254
- allow_detached_data: bool = True,
1255
- allow_detached_table: bool = False,
1256
- check_db: bool = True,
1257
- check_logical: bool = True,
1258
- check_field: bool = True,
1259
- enable_pre_save: bool = True,
1260
- enable_post_save: bool = True,
1251
+ self,
1252
+ data: Union[pd.DataFrame, Dict[str, pd.DataFrame]],
1253
+ primary_kv: Union[Any, Dict[str, Any]],
1254
+ parent_kv: Union[Any, Dict[str, Any]] = None,
1255
+ allow_detached_data: bool = True,
1256
+ allow_detached_table: bool = False,
1257
+ check_db: bool = True,
1258
+ check_logical: bool = True,
1259
+ check_field: bool = True,
1260
+ enable_pre_save: bool = True,
1261
+ enable_post_save: bool = True,
1261
1262
  ) -> List[ModelDataReturnDTO]:
1262
1263
  ...
1264
+
1265
+ def copy_rows(
1266
+ self,
1267
+ config: Union[CopyConfig, Dict]
1268
+ ):
1269
+ ...
@@ -1,3 +1,4 @@
1
+ import re
1
2
  import warnings
2
3
  from contextvars import ContextVar
3
4
  import pprint
@@ -58,6 +59,10 @@ __all__ = [
58
59
  'Skip',
59
60
  'Field',
60
61
  'get_table_class',
62
+ 'T_DatatableClass',
63
+ 'T_DatatableInstance',
64
+ 'T_AsyncDatatableClass',
65
+ 'T_AsyncDatatableInstance'
61
66
  ]
62
67
 
63
68
  SQL_LOG_MAX_LEN = 1024
@@ -933,7 +938,10 @@ class AsyncDataTableMySQL(ElementBase):
933
938
  if isinstance(c, str):
934
939
  res.append(c)
935
940
  elif isinstance(c, Term):
936
- res.append(c.get_sql(quote_char=''))
941
+ if c.alias is not None:
942
+ res.append(c.alias)
943
+ else:
944
+ res.append(c.get_sql(quote_char=''))
937
945
  else:
938
946
  res.append(str(c))
939
947
  return res
@@ -1963,7 +1971,7 @@ class _OracleField(Field):
1963
1971
  quote_char = kwargs.pop("quote_char", '"')
1964
1972
 
1965
1973
  field_sql = format_quotes(self.name, quote_char)
1966
-
1974
+ field_sql = field_sql.upper()
1967
1975
  # Need to add namespace if the table has an alias
1968
1976
  if self.table and (with_namespace or self.table.alias):
1969
1977
  table_name = self.table.get_table_name()
@@ -1972,7 +1980,6 @@ class _OracleField(Field):
1972
1980
  name=field_sql,
1973
1981
  )
1974
1982
 
1975
- field_sql = field_sql.upper()
1976
1983
  field_alias = getattr(self, "alias", None)
1977
1984
  if with_alias:
1978
1985
  return format_alias_sql(field_sql, field_alias, quote_char=quote_char, **kwargs)
@@ -1981,7 +1988,7 @@ class _OracleField(Field):
1981
1988
 
1982
1989
  class OracleTable(Table):
1983
1990
  def field(self, name: str) -> Field:
1984
- return _OracleField(name, table=self)
1991
+ return _OracleField(name, table=self, alias=name)
1985
1992
 
1986
1993
 
1987
1994
  class AsyncDataTableOracle(AsyncDataTableMySQL):
@@ -2265,19 +2272,38 @@ class DataTableDeepEngine(
2265
2272
  raise NotImplementedError('DeepEngine does not support transaction.')
2266
2273
 
2267
2274
 
2268
- def get_table_class(
2269
- element_type: str,
2270
- sync: bool = True
2271
- ) -> Union[
2272
- Type[AsyncDataTableMySQL],
2273
- Type[AsyncDataTableClickHouse],
2274
- Type[AsyncDataTableOracle],
2275
- Type[AsyncDataTableSQLServer],
2276
- Type[AsyncDataTableKingBase],
2277
- Type[AsyncDataTableGauss],
2278
- Type[AsyncDataTableDaMeng],
2279
- Type[AsyncDataTablePostgreSQL],
2280
- Type[AsyncDataTableDeepEngine],
2275
+ _RE_PARSE_SERVER = re.compile(r"data[-]?table-(.*?)-server[\d]-[\d]")
2276
+
2277
+
2278
+ TO_MODULE_TYPE = CaseInsensitiveDict(
2279
+ {
2280
+ 'mysql': MySQLAPI.module_type,
2281
+ 'clickhouse': ClickHouseAPI.module_type,
2282
+ 'sqlserver': SQLServerAPI.module_type,
2283
+ 'oracle': OracleAPI.module_type,
2284
+ 'kingbase': KingBaseAPI.module_type,
2285
+ 'gauss': GaussAPI.module_type,
2286
+ 'dameng': DaMengAPI.module_type,
2287
+ 'postgresql': PostgreSQLAPI.module_type,
2288
+ 'deepengine': DeepEngineAPI.module_type,
2289
+ }
2290
+ )
2291
+
2292
+ TABLE = CaseInsensitiveDict(
2293
+ {
2294
+ MySQLAPI.module_type: (DataTableMySQL, AsyncDataTableMySQL),
2295
+ ClickHouseAPI.module_type: (DataTableClickHouse, AsyncDataTableClickHouse),
2296
+ SQLServerAPI.module_type: (DataTableSQLServer, AsyncDataTableSQLServer),
2297
+ OracleAPI.module_type: (DataTableOracle, AsyncDataTableOracle),
2298
+ KingBaseAPI.module_type: (DataTableKingBase, AsyncDataTableKingBase),
2299
+ GaussAPI.module_type: (DataTableGauss, AsyncDataTableGauss),
2300
+ DaMengAPI.module_type: (DataTableDaMeng, AsyncDataTableDaMeng),
2301
+ PostgreSQLAPI.module_type: (DataTablePostgreSQL, AsyncDataTablePostgreSQL),
2302
+ DeepEngineAPI.module_type: (DataTableDeepEngine, AsyncDataTableDeepEngine),
2303
+ }
2304
+ )
2305
+
2306
+ T_DatatableClass = Union[
2281
2307
  Type[DataTableMySQL],
2282
2308
  Type[DataTableClickHouse],
2283
2309
  Type[DataTableOracle],
@@ -2286,13 +2312,58 @@ def get_table_class(
2286
2312
  Type[DataTableGauss],
2287
2313
  Type[DataTableDaMeng],
2288
2314
  Type[DataTablePostgreSQL],
2289
- Type[DataTableDeepEngine],
2315
+ Type[DataTableDeepEngine]
2316
+ ]
2317
+
2318
+ T_AsyncDatatableClass = Union[
2319
+ Type[AsyncDataTableMySQL],
2320
+ Type[AsyncDataTableClickHouse],
2321
+ Type[AsyncDataTableOracle],
2322
+ Type[AsyncDataTableSQLServer],
2323
+ Type[AsyncDataTableKingBase],
2324
+ Type[AsyncDataTableGauss],
2325
+ Type[AsyncDataTableDaMeng],
2326
+ Type[AsyncDataTablePostgreSQL],
2327
+ Type[AsyncDataTableDeepEngine]
2328
+ ]
2329
+
2330
+ T_DatatableInstance = Union[
2331
+ DataTableMySQL,
2332
+ DataTableClickHouse,
2333
+ DataTableOracle,
2334
+ DataTableSQLServer,
2335
+ DataTableKingBase,
2336
+ DataTableGauss,
2337
+ DataTableDaMeng,
2338
+ DataTablePostgreSQL,
2339
+ DataTableDeepEngine
2340
+ ]
2341
+
2342
+ T_AsyncDatatableInstance = Union[
2343
+ AsyncDataTableMySQL,
2344
+ AsyncDataTableClickHouse,
2345
+ AsyncDataTableOracle,
2346
+ AsyncDataTableSQLServer,
2347
+ AsyncDataTableKingBase,
2348
+ AsyncDataTableGauss,
2349
+ AsyncDataTableDaMeng,
2350
+ AsyncDataTablePostgreSQL,
2351
+ AsyncDataTableDeepEngine
2352
+ ]
2353
+
2354
+
2355
+ def get_table_class(
2356
+ element_type: str,
2357
+ sync: bool = True
2358
+ ) -> Union[
2359
+ T_DatatableClass,
2360
+ T_AsyncDatatableClass
2290
2361
  ]:
2291
2362
  """
2292
2363
  根据元素类型获取对应的数据表元素类
2293
2364
 
2294
2365
  Args:
2295
- element_type: 元素类型
2366
+ element_type: module type或server name
2296
2367
  sync: 同步或异步元素类,默认同步
2297
2368
 
2298
2369
  """
@@ -2301,19 +2372,21 @@ def get_table_class(
2301
2372
  else:
2302
2373
  index = 1
2303
2374
 
2304
- table = {
2305
- MySQLAPI.module_type: (DataTableMySQL, AsyncDataTableMySQL),
2306
- ClickHouseAPI.module_type: (DataTableClickHouse, AsyncDataTableClickHouse),
2307
- SQLServerAPI.module_type: (DataTableSQLServer, AsyncDataTableSQLServer),
2308
- OracleAPI.module_type: (DataTableOracle, AsyncDataTableOracle),
2309
- KingBaseAPI.module_type: (DataTableKingBase, AsyncDataTableKingBase),
2310
- GaussAPI.module_type: (DataTableGauss, AsyncDataTableGauss),
2311
- DaMengAPI.module_type: (DataTableDaMeng, AsyncDataTableDaMeng),
2312
- PostgreSQLAPI.module_type: (DataTablePostgreSQL, AsyncDataTablePostgreSQL),
2313
- DeepEngineAPI.module_type: (DataTableDeepEngine, AsyncDataTableDeepEngine),
2314
- }.get(element_type.upper())
2375
+ if element_type is None:
2376
+ raise ValueError("`element_type` should be a string value.")
2377
+
2378
+ module_type = element_type
2379
+
2380
+ if match := _RE_PARSE_SERVER.match(element_type):
2381
+ server_name = match.group(1)
2382
+ module_type = TO_MODULE_TYPE.get(server_name)
2383
+
2384
+ if module_type is None:
2385
+ raise ValueError(f"{element_type} is not a known datatable server.")
2386
+
2387
+ table = TABLE.get(module_type)
2315
2388
 
2316
2389
  if table is None:
2317
- raise TypeError(f"Unknown element type: {element_type}")
2318
- else:
2319
- return table[index]
2390
+ raise TypeError(f"Unknown datatable type: {element_type}")
2391
+
2392
+ return table[index]