deepfos 1.1.60__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. deepfos/__init__.py +6 -0
  2. deepfos/_version.py +21 -0
  3. deepfos/algo/__init__.py +0 -0
  4. deepfos/algo/graph.py +171 -0
  5. deepfos/algo/segtree.py +31 -0
  6. deepfos/api/V1_1/__init__.py +0 -0
  7. deepfos/api/V1_1/business_model.py +119 -0
  8. deepfos/api/V1_1/dimension.py +599 -0
  9. deepfos/api/V1_1/models/__init__.py +0 -0
  10. deepfos/api/V1_1/models/business_model.py +1033 -0
  11. deepfos/api/V1_1/models/dimension.py +2768 -0
  12. deepfos/api/V1_2/__init__.py +0 -0
  13. deepfos/api/V1_2/dimension.py +285 -0
  14. deepfos/api/V1_2/models/__init__.py +0 -0
  15. deepfos/api/V1_2/models/dimension.py +2923 -0
  16. deepfos/api/__init__.py +0 -0
  17. deepfos/api/account.py +167 -0
  18. deepfos/api/accounting_engines.py +147 -0
  19. deepfos/api/app.py +626 -0
  20. deepfos/api/approval_process.py +198 -0
  21. deepfos/api/base.py +983 -0
  22. deepfos/api/business_model.py +160 -0
  23. deepfos/api/consolidation.py +129 -0
  24. deepfos/api/consolidation_process.py +106 -0
  25. deepfos/api/datatable.py +341 -0
  26. deepfos/api/deep_pipeline.py +61 -0
  27. deepfos/api/deepconnector.py +36 -0
  28. deepfos/api/deepfos_task.py +92 -0
  29. deepfos/api/deepmodel.py +188 -0
  30. deepfos/api/dimension.py +486 -0
  31. deepfos/api/financial_model.py +319 -0
  32. deepfos/api/journal_model.py +119 -0
  33. deepfos/api/journal_template.py +132 -0
  34. deepfos/api/memory_financial_model.py +98 -0
  35. deepfos/api/models/__init__.py +3 -0
  36. deepfos/api/models/account.py +483 -0
  37. deepfos/api/models/accounting_engines.py +756 -0
  38. deepfos/api/models/app.py +1338 -0
  39. deepfos/api/models/approval_process.py +1043 -0
  40. deepfos/api/models/base.py +234 -0
  41. deepfos/api/models/business_model.py +805 -0
  42. deepfos/api/models/consolidation.py +711 -0
  43. deepfos/api/models/consolidation_process.py +248 -0
  44. deepfos/api/models/datatable_mysql.py +427 -0
  45. deepfos/api/models/deep_pipeline.py +55 -0
  46. deepfos/api/models/deepconnector.py +28 -0
  47. deepfos/api/models/deepfos_task.py +386 -0
  48. deepfos/api/models/deepmodel.py +308 -0
  49. deepfos/api/models/dimension.py +1576 -0
  50. deepfos/api/models/financial_model.py +1796 -0
  51. deepfos/api/models/journal_model.py +341 -0
  52. deepfos/api/models/journal_template.py +854 -0
  53. deepfos/api/models/memory_financial_model.py +478 -0
  54. deepfos/api/models/platform.py +178 -0
  55. deepfos/api/models/python.py +221 -0
  56. deepfos/api/models/reconciliation_engine.py +411 -0
  57. deepfos/api/models/reconciliation_report.py +161 -0
  58. deepfos/api/models/role_strategy.py +884 -0
  59. deepfos/api/models/smartlist.py +237 -0
  60. deepfos/api/models/space.py +1137 -0
  61. deepfos/api/models/system.py +1065 -0
  62. deepfos/api/models/variable.py +463 -0
  63. deepfos/api/models/workflow.py +946 -0
  64. deepfos/api/platform.py +199 -0
  65. deepfos/api/python.py +90 -0
  66. deepfos/api/reconciliation_engine.py +181 -0
  67. deepfos/api/reconciliation_report.py +64 -0
  68. deepfos/api/role_strategy.py +234 -0
  69. deepfos/api/smartlist.py +69 -0
  70. deepfos/api/space.py +582 -0
  71. deepfos/api/system.py +372 -0
  72. deepfos/api/variable.py +154 -0
  73. deepfos/api/workflow.py +264 -0
  74. deepfos/boost/__init__.py +6 -0
  75. deepfos/boost/py_jstream.py +89 -0
  76. deepfos/boost/py_pandas.py +20 -0
  77. deepfos/cache.py +121 -0
  78. deepfos/config.py +6 -0
  79. deepfos/core/__init__.py +27 -0
  80. deepfos/core/cube/__init__.py +10 -0
  81. deepfos/core/cube/_base.py +462 -0
  82. deepfos/core/cube/constants.py +21 -0
  83. deepfos/core/cube/cube.py +408 -0
  84. deepfos/core/cube/formula.py +707 -0
  85. deepfos/core/cube/syscube.py +532 -0
  86. deepfos/core/cube/typing.py +7 -0
  87. deepfos/core/cube/utils.py +238 -0
  88. deepfos/core/dimension/__init__.py +11 -0
  89. deepfos/core/dimension/_base.py +506 -0
  90. deepfos/core/dimension/dimcreator.py +184 -0
  91. deepfos/core/dimension/dimension.py +472 -0
  92. deepfos/core/dimension/dimexpr.py +271 -0
  93. deepfos/core/dimension/dimmember.py +155 -0
  94. deepfos/core/dimension/eledimension.py +22 -0
  95. deepfos/core/dimension/filters.py +99 -0
  96. deepfos/core/dimension/sysdimension.py +168 -0
  97. deepfos/core/logictable/__init__.py +5 -0
  98. deepfos/core/logictable/_cache.py +141 -0
  99. deepfos/core/logictable/_operator.py +663 -0
  100. deepfos/core/logictable/nodemixin.py +673 -0
  101. deepfos/core/logictable/sqlcondition.py +609 -0
  102. deepfos/core/logictable/tablemodel.py +497 -0
  103. deepfos/db/__init__.py +36 -0
  104. deepfos/db/cipher.py +660 -0
  105. deepfos/db/clickhouse.py +191 -0
  106. deepfos/db/connector.py +195 -0
  107. deepfos/db/daclickhouse.py +171 -0
  108. deepfos/db/dameng.py +101 -0
  109. deepfos/db/damysql.py +189 -0
  110. deepfos/db/dbkits.py +358 -0
  111. deepfos/db/deepengine.py +99 -0
  112. deepfos/db/deepmodel.py +82 -0
  113. deepfos/db/deepmodel_kingbase.py +83 -0
  114. deepfos/db/edb.py +214 -0
  115. deepfos/db/gauss.py +83 -0
  116. deepfos/db/kingbase.py +83 -0
  117. deepfos/db/mysql.py +184 -0
  118. deepfos/db/oracle.py +131 -0
  119. deepfos/db/postgresql.py +192 -0
  120. deepfos/db/sqlserver.py +99 -0
  121. deepfos/db/utils.py +135 -0
  122. deepfos/element/__init__.py +89 -0
  123. deepfos/element/accounting.py +348 -0
  124. deepfos/element/apvlprocess.py +215 -0
  125. deepfos/element/base.py +398 -0
  126. deepfos/element/bizmodel.py +1269 -0
  127. deepfos/element/datatable.py +2467 -0
  128. deepfos/element/deep_pipeline.py +186 -0
  129. deepfos/element/deepconnector.py +59 -0
  130. deepfos/element/deepmodel.py +1806 -0
  131. deepfos/element/dimension.py +1254 -0
  132. deepfos/element/fact_table.py +427 -0
  133. deepfos/element/finmodel.py +1485 -0
  134. deepfos/element/journal.py +840 -0
  135. deepfos/element/journal_template.py +943 -0
  136. deepfos/element/pyscript.py +412 -0
  137. deepfos/element/reconciliation.py +553 -0
  138. deepfos/element/rolestrategy.py +243 -0
  139. deepfos/element/smartlist.py +457 -0
  140. deepfos/element/variable.py +756 -0
  141. deepfos/element/workflow.py +560 -0
  142. deepfos/exceptions/__init__.py +239 -0
  143. deepfos/exceptions/hook.py +86 -0
  144. deepfos/lazy.py +104 -0
  145. deepfos/lazy_import.py +84 -0
  146. deepfos/lib/__init__.py +0 -0
  147. deepfos/lib/_javaobj.py +366 -0
  148. deepfos/lib/asynchronous.py +879 -0
  149. deepfos/lib/concurrency.py +107 -0
  150. deepfos/lib/constant.py +39 -0
  151. deepfos/lib/decorator.py +310 -0
  152. deepfos/lib/deepchart.py +778 -0
  153. deepfos/lib/deepux.py +477 -0
  154. deepfos/lib/discovery.py +273 -0
  155. deepfos/lib/edb_lexer.py +789 -0
  156. deepfos/lib/eureka.py +156 -0
  157. deepfos/lib/filterparser.py +751 -0
  158. deepfos/lib/httpcli.py +106 -0
  159. deepfos/lib/jsonstreamer.py +80 -0
  160. deepfos/lib/msg.py +394 -0
  161. deepfos/lib/nacos.py +225 -0
  162. deepfos/lib/patch.py +92 -0
  163. deepfos/lib/redis.py +241 -0
  164. deepfos/lib/serutils.py +181 -0
  165. deepfos/lib/stopwatch.py +99 -0
  166. deepfos/lib/subtask.py +572 -0
  167. deepfos/lib/sysutils.py +703 -0
  168. deepfos/lib/utils.py +1003 -0
  169. deepfos/local.py +160 -0
  170. deepfos/options.py +670 -0
  171. deepfos/translation.py +237 -0
  172. deepfos-1.1.60.dist-info/METADATA +33 -0
  173. deepfos-1.1.60.dist-info/RECORD +175 -0
  174. deepfos-1.1.60.dist-info/WHEEL +5 -0
  175. deepfos-1.1.60.dist-info/top_level.txt +1 -0
@@ -0,0 +1,879 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import itertools
5
+ import asyncio.events as events
6
+ from asyncio import Future as AioFuture
7
+ import sys
8
+ from contextlib import contextmanager
9
+ from heapq import heappop
10
+ import functools
11
+ import inspect
12
+ import threading
13
+ import time
14
+ import weakref
15
+ import atexit
16
+ from reprlib import aRepr
17
+ from typing import (
18
+ Optional, Tuple, Union, TYPE_CHECKING, List,
19
+ Callable, NamedTuple, Coroutine, Any, Dict,
20
+ Generic, Iterable, TypeVar, Awaitable, overload, Container, Set,
21
+ )
22
+ # noinspection PyProtectedMember
23
+ from concurrent.futures._base import (
24
+ Future, TimeoutError as FutureTimeout
25
+ )
26
+
27
+ from loguru import logger
28
+
29
+ from deepfos.options import OPTION
30
+ from deepfos.exceptions import BadFutureError, eliminate_from_traceback
31
+ from .constant import UNSET
32
+
33
+ aRepr.maxstring = 200
34
+
35
+ __all__ = [
36
+ 'evloop',
37
+ 'deepfosio',
38
+ 'register_on_loop_shutdown',
39
+ 'async_to_sync',
40
+ 'future_property',
41
+ 'FuturePropertyMeta',
42
+ 'cache_async',
43
+ ]
44
+
45
+ eliminate_from_traceback(__file__)
46
+ eliminate_from_traceback(Future)
47
+
48
+ MAIN_TID = threading.get_ident()
49
+ _LATER_THAN_3_10 = sys.version_info >= (3, 10)
50
+
51
+ CoroutineFunc = Callable[[], Coroutine]
52
+ NodeT = TypeVar('NodeT')
53
+ LinkListItem = TypeVar('LinkListItem')
54
+ ReturnT = TypeVar('ReturnT')
55
+ FutureT = TypeVar('FutureT', bound=Union[Future, asyncio.Future])
56
+ FP = TypeVar('FP', bound='future_property')
57
+ CallbacksT = List[Tuple[Callable, Union[bool, object]]]
58
+
59
+
60
+ # -----------------------------------------------------------------------------
61
+ # make Reentrant
62
+ # noinspection PyUnresolvedReferences,PyProtectedMember
63
+ def _patch_task():
64
+ """Patch the Task's step and enter/leave methods to make it reentrant."""
65
+ asyncio.Task = asyncio.tasks._CTask = asyncio.tasks.Task = \
66
+ asyncio.tasks._PyTask
67
+ asyncio.Future = asyncio.futures._CFuture = asyncio.futures.Future = \
68
+ asyncio.futures._PyFuture
69
+
70
+ eliminate_from_traceback(asyncio.Future)
71
+ eliminate_from_traceback(asyncio.Task)
72
+
73
+ Task = asyncio.Task # noqa
74
+ if getattr(Task, '__deepfos_patched__', False):
75
+ return
76
+
77
+ # noinspection PyProtectedMember
78
+ def step(task, exc=None):
79
+ curr_task = curr_tasks.get(task._loop)
80
+ try:
81
+ step_orig(task, exc)
82
+ finally:
83
+ if curr_task is None:
84
+ curr_tasks.pop(task._loop, None)
85
+ else:
86
+ curr_tasks[task._loop] = curr_task
87
+
88
+ def enter_task(loop, task):
89
+ curr_tasks[loop] = task
90
+
91
+ def leave_task(loop, task): # noqa
92
+ curr_tasks.pop(loop, None)
93
+
94
+ asyncio.tasks._enter_task = enter_task
95
+ asyncio.tasks._leave_task = leave_task
96
+ curr_tasks = asyncio.tasks._current_tasks
97
+ step_orig = Task._Task__step
98
+ Task._Task__step = step
99
+ setattr(Task, '__deepfos_patched__', True)
100
+
101
+
102
+ _patch_task()
103
+
104
+ if sys.platform == 'win32':
105
+ BaseEventLoop = asyncio.ProactorEventLoop
106
+ is_proactorloop = True
107
+ else:
108
+ BaseEventLoop = asyncio.SelectorEventLoop
109
+ is_proactorloop = False
110
+
111
+
112
+ # noinspection PyUnresolvedReferences,PyAttributeOutsideInit,PyProtectedMember
113
+ class ReentrantLoop(BaseEventLoop):
114
+ _num_runs_pending = 0
115
+ _is_proactorloop = is_proactorloop
116
+ _nest_patched = True
117
+
118
+ def run_forever(self):
119
+ with self.manage_run(), self.manage_asyncgens():
120
+ while True:
121
+ self._run_once()
122
+ if self._stopping:
123
+ break
124
+ self._stopping = False
125
+
126
+ def run_until_complete(self, future):
127
+ with self.manage_run():
128
+ f = asyncio.ensure_future(future, loop=self)
129
+ if f is not future:
130
+ f._log_destroy_pending = False
131
+ while not f.done():
132
+ self._run_once()
133
+ if self._stopping:
134
+ break # pragma: no cover
135
+ if not f.done(): # pragma: no cover
136
+ raise RuntimeError(
137
+ 'Event loop stopped before Future completed.')
138
+ return f.result()
139
+
140
+ @contextmanager
141
+ def manage_run(self):
142
+ """Set up the loop for running."""
143
+ self._check_closed()
144
+ old_thread_id = self._thread_id
145
+ old_running_loop = events._get_running_loop()
146
+ try:
147
+ self._thread_id = threading.get_ident()
148
+ events._set_running_loop(self)
149
+ self._num_runs_pending += 1
150
+ if self._is_proactorloop:
151
+ if self._self_reading_future is None:
152
+ self.call_soon(self._loop_self_reading)
153
+ yield
154
+ finally:
155
+ self._thread_id = old_thread_id
156
+ events._set_running_loop(old_running_loop)
157
+ self._num_runs_pending -= 1
158
+ if self._is_proactorloop:
159
+ if (self._num_runs_pending == 0
160
+ and self._self_reading_future is not None):
161
+ ov = self._self_reading_future._ov
162
+ self._self_reading_future.cancel()
163
+ if ov is not None:
164
+ self._proactor._unregister(ov)
165
+ self._self_reading_future = None
166
+
167
+ @contextmanager
168
+ def manage_asyncgens(self):
169
+ if not hasattr(sys, 'get_asyncgen_hooks'):
170
+ # Python version is too old.
171
+ return # pragma: no cover
172
+ old_agen_hooks = sys.get_asyncgen_hooks()
173
+ try:
174
+ self._set_coroutine_origin_tracking(self._debug)
175
+ if self._asyncgens is not None:
176
+ sys.set_asyncgen_hooks(
177
+ firstiter=self._asyncgen_firstiter_hook,
178
+ finalizer=self._asyncgen_finalizer_hook)
179
+ yield
180
+ finally:
181
+ self._set_coroutine_origin_tracking(False)
182
+ if self._asyncgens is not None:
183
+ sys.set_asyncgen_hooks(*old_agen_hooks)
184
+
185
+ def _check_running(self): # pragma: no cover
186
+ """Do not throw exception if loop is already running."""
187
+ pass
188
+
189
+ def _run_once(self):
190
+ """
191
+ Simplified re-implementation of asyncio's _run_once that
192
+ runs handles as they become ready.
193
+ """
194
+ ready = self._ready
195
+ scheduled = self._scheduled
196
+ while scheduled and scheduled[0]._cancelled:
197
+ heappop(scheduled)
198
+
199
+ timeout = (
200
+ 0 if ready or self._stopping
201
+ else min(max(
202
+ scheduled[0]._when - self.time(), 0), 86400) if scheduled
203
+ else None)
204
+ event_list = self._selector.select(timeout)
205
+ self._process_events(event_list)
206
+
207
+ end_time = self.time() + self._clock_resolution
208
+ while scheduled and scheduled[0]._when < end_time: # pragma: no cover
209
+ handle = heappop(scheduled)
210
+ ready.append(handle)
211
+
212
+ for _ in range(len(ready)):
213
+ if not ready: # pragma: no cover
214
+ break
215
+ handle = ready.popleft()
216
+ if not handle._cancelled:
217
+ handle._run()
218
+ handle = None # noqa
219
+
220
+
221
+ class ReentrantLoopPolicy(events.BaseDefaultEventLoopPolicy): # noqa
222
+ _loop_factory = ReentrantLoop
223
+
224
+
225
+ # -----------------------------------------------------------------------------
226
+ # core
227
+ class Node(Generic[NodeT]):
228
+ __slots__ = ('next', 'value', '_prev', '__weakref__')
229
+ if TYPE_CHECKING:
230
+ next: Optional['Node[NodeT]']
231
+
232
+ def __init__(self, value: Optional[NodeT] = None):
233
+ self.value = value
234
+ self.next = None
235
+ self._prev = None
236
+
237
+ @property
238
+ def prev(self):
239
+ if not self._prev: # pragma: no cover
240
+ return self._prev
241
+ return self._prev()
242
+
243
+ @prev.setter
244
+ def prev(self, value):
245
+ if value is None: # pragma: no cover
246
+ self._prev = None
247
+ else:
248
+ self._prev = weakref.ref(value)
249
+
250
+ def __bool__(self):
251
+ return True
252
+
253
+ def __repr__(self): # pragma: no cover
254
+ return repr(self.value)
255
+
256
+
257
+ class LinkedList(Generic[LinkListItem]):
258
+ def __init__(self):
259
+ self.tail = self.root = Node()
260
+
261
+ def __len__(self):
262
+ return sum(1 for _ in self)
263
+
264
+ def append(self, node: Node[LinkListItem]):
265
+ self.tail.next = node
266
+ node.prev = self.tail
267
+ self.tail = node
268
+
269
+ def wrap_append(self, value: LinkListItem):
270
+ node = Node(value)
271
+ self.append(node)
272
+ return node
273
+
274
+ def remove(self, node: Node[LinkListItem]):
275
+ _prev = node.prev
276
+ _next = node.next
277
+
278
+ _prev.next = _next
279
+
280
+ if _next is None:
281
+ self.tail = _prev
282
+ else:
283
+ _next.prev = _prev
284
+
285
+ def __repr__(self):
286
+ return "[{}]".format(', '.join(repr(item) for item in self))
287
+
288
+ def __iter__(self) -> Iterable[LinkListItem]:
289
+ node = self.root
290
+ while node := node.next:
291
+ yield node.value
292
+
293
+ def clear(self):
294
+ node = self.root
295
+ while node := node.next:
296
+ self.remove(node)
297
+
298
+
299
+ class _AbstractTask(Generic[ReturnT]):
300
+ def __init__(
301
+ self,
302
+ future: Union[Future[ReturnT], AioFuture[ReturnT]],
303
+ coro: Coroutine[ReturnT, Any, ReturnT],
304
+ ensure_completed: bool = False,
305
+ die_in_peace: bool = False
306
+ ):
307
+ self.future = future
308
+ self._display_name = coro.__qualname__
309
+ self.ensure_completed = ensure_completed
310
+ self.die_in_peace = die_in_peace
311
+
312
+ def result(self, timeout: Optional[float] = None) -> ReturnT: # pragma: no cover
313
+ raise NotImplementedError
314
+
315
+ def done(self) -> bool:
316
+ return self.future.done()
317
+
318
+ def add_done_callback(self, callback):
319
+ return self.future.add_done_callback(callback)
320
+
321
+ def cancel(self):
322
+ return self.future.cancel()
323
+
324
+ def __repr__(self):
325
+ if self.ensure_completed:
326
+ lb, rb = '{{', '}}'
327
+ else:
328
+ lb, rb = '<', '>'
329
+ return (
330
+ f"{lb}{self.__class__.__name__} "
331
+ f"{self._display_name} at 0x{hex(id(self))}{rb}"
332
+ )
333
+
334
+
335
+ class AsyncIOTask(_AbstractTask):
336
+ def __init__(
337
+ self,
338
+ future: AioFuture[ReturnT],
339
+ coro: Coroutine[ReturnT, Any, ReturnT],
340
+ ensure_completed: bool = False,
341
+ ):
342
+ super().__init__(future, coro, ensure_completed)
343
+ self.r = UNSET
344
+ self.exc = None
345
+ self.future.add_done_callback(self.set_task_done)
346
+
347
+ def set_task_done(self, future: AioFuture):
348
+ if exc := future.exception():
349
+ self.exc = exc
350
+ else:
351
+ self.r = future.result()
352
+
353
+ async def _wait_for_result(self, timeout=None):
354
+ return await asyncio.wait_for(self.future, timeout)
355
+
356
+ def result(self, timeout: Optional[float] = None) -> ReturnT:
357
+ if self.exc is not None:
358
+ raise self.exc
359
+ if self.r is UNSET:
360
+ evloop.run(self._wait_for_result(timeout))
361
+ return self.r
362
+
363
+
364
+ class ThreadTask(_AbstractTask):
365
+ if TYPE_CHECKING:
366
+ future: Future
367
+
368
+ def result(self, timeout: Optional[float] = None) -> ReturnT:
369
+ if (
370
+ evloop.in_same_thread()
371
+ and not (
372
+ self.future.done()
373
+ or self.future.running()
374
+ )
375
+ ):
376
+ raise RuntimeError(
377
+ "Waiting a pending task in "
378
+ "same thread will lead to deadlock!"
379
+ )
380
+
381
+ return self.future.result(timeout=timeout)
382
+
383
+
384
+ class _PeriodicTask(NamedTuple):
385
+ task: _AbstractTask
386
+ on_stop: Optional[CoroutineFunc]
387
+
388
+
389
+ class EventLoop:
390
+
391
+ class _Loop(threading.Thread):
392
+ counter = itertools.count().__next__
393
+
394
+ def __init__(self, bind):
395
+ threading.Thread.__init__(self, name=f"evloop-{self.counter()}")
396
+ self.daemon = True
397
+ self.id = None
398
+ self._bind = weakref.ref(bind)
399
+ default_policy = asyncio.get_event_loop_policy()
400
+ try:
401
+ asyncio.set_event_loop_policy(ReentrantLoopPolicy())
402
+ self.loop = asyncio.new_event_loop()
403
+ finally:
404
+ asyncio.set_event_loop_policy(default_policy)
405
+
406
+ def run(self):
407
+ try:
408
+ loop = self.loop
409
+ self._bind().set_loop(loop)
410
+ self.id = threading.get_ident()
411
+ logger.debug(f"Starting loop in thread: [{self.id}]")
412
+ asyncio.set_event_loop(loop)
413
+ loop.run_forever()
414
+ logger.debug(f"Loop in thread: [{self.id}] stopped.")
415
+ except (KeyboardInterrupt, SystemExit): # pragma: no cover
416
+ try:
417
+ import _thread as thread
418
+ except ImportError:
419
+ import thread # noqa
420
+ thread.interrupt_main()
421
+ except Exception: # pragma: no cover
422
+ logger.exception("Loop stopped due to unexpected error.")
423
+ raise
424
+
425
+ def stop(self):
426
+ self.loop.call_soon_threadsafe(self.loop.stop)
427
+ time.sleep(0.001)
428
+ self.join()
429
+
430
+ def __init__(self):
431
+ self.__thread = None
432
+ self.__loop = None
433
+ self.__lock = threading.Lock()
434
+ self._tasks = LinkedList[_AbstractTask]()
435
+ atexit.register(self.stop)
436
+ self._setup()
437
+ self._on_shutdown = []
438
+ self._periodic_task: List[_PeriodicTask] = []
439
+
440
+ @property
441
+ def loop(self) -> asyncio.BaseEventLoop:
442
+ if self.__loop is None:
443
+ with self.__lock:
444
+ if self.__loop is None:
445
+ self._setup()
446
+ elif self.__loop.is_closed(): # pragma: no cover
447
+ with self.__lock:
448
+ if self.__loop.is_closed():
449
+ self._setup()
450
+ return self.__loop
451
+
452
+ @property
453
+ def thread(self):
454
+ if self.__thread is None:
455
+ with self.__lock:
456
+ if self.__thread is None:
457
+ self._setup()
458
+ return self.__thread
459
+
460
+ def _setup(self):
461
+ self.stop()
462
+ self.__thread = self._Loop(self)
463
+ self.__thread.start()
464
+
465
+ def set_loop(self, loop: asyncio.BaseEventLoop):
466
+ self.__loop = loop
467
+
468
+ def run(self, coro: Coroutine[ReturnT, Any, ReturnT]) -> ReturnT:
469
+ if self.in_same_thread():
470
+ return self.loop.run_until_complete(coro)
471
+ else:
472
+ return self.create_task(coro).result()
473
+
474
+ def create_task(
475
+ self,
476
+ coro: Coroutine[ReturnT, Any, ReturnT],
477
+ ensure_completed=False
478
+ ) -> _AbstractTask[ReturnT]:
479
+ task: _AbstractTask
480
+ if self.in_same_thread():
481
+ task = AsyncIOTask(
482
+ asyncio.create_task(coro),
483
+ coro, ensure_completed
484
+ )
485
+ else:
486
+ task = ThreadTask(
487
+ asyncio.run_coroutine_threadsafe(coro, self.loop),
488
+ coro, ensure_completed
489
+ )
490
+ logger.debug(f"Task {task!r} created.")
491
+ node = self._tasks.wrap_append(task)
492
+ task.add_done_callback(self._report_task_done(node))
493
+ return task
494
+
495
+ apply = create_task
496
+
497
+ def _report_task_done(self, node: Node):
498
+ def cb(fut):
499
+ def get_hint():
500
+ if exc := fut.exception():
501
+ return f'failed with exception: {exc!r}'
502
+ else:
503
+ return f'finished with result: {aRepr.repr(fut.result())}'
504
+ logger.opt(lazy=True).debug(f"Task {node.value} {{}}", get_hint)
505
+ self._tasks.remove(node)
506
+ return cb
507
+
508
+ def register_shutdown(self, handler, is_coro=UNSET):
509
+ self._on_shutdown.append((handler, is_coro))
510
+
511
+ def stop(self):
512
+ if self.__loop is None:
513
+ return
514
+
515
+ for periodic_task in self._periodic_task:
516
+ self.cancel_periodic_task(periodic_task)
517
+
518
+ default_timeout = OPTION.general.coro_graceful_timeout
519
+ self.run(DeepFOSIO.apply_handler(self._on_shutdown))
520
+
521
+ for task in self._tasks:
522
+ # noinspection PyBroadException
523
+ try:
524
+ if not task.done():
525
+ timeout = None if task.ensure_completed else default_timeout
526
+ if isinstance(task, AsyncIOTask):
527
+ self.__loop.call_soon_threadsafe(task.result, timeout)
528
+ else:
529
+ task.result(timeout)
530
+ except FutureTimeout: # pragma: no cover
531
+ if not task.die_in_peace:
532
+ logger.error(f"Wait for task: {task} timed out.")
533
+ except Exception: # pragma: no cover
534
+ if not task.die_in_peace:
535
+ logger.exception("Failed to wait task.")
536
+
537
+ waited = 0
538
+ interval = 0.05
539
+ while (
540
+ (n := len(asyncio.all_tasks(loop=self.__loop))) > 0
541
+ and waited < default_timeout
542
+ ):
543
+ time.sleep(interval)
544
+ waited += interval
545
+ interval *= 2
546
+ logger.debug(f"Still waiting {n} tasks to finish..")
547
+
548
+ if self.__thread is not None:
549
+ logger.debug(f"Stopping loop in thread: [{self.__thread.id}] ...")
550
+ self.__thread.stop()
551
+ self.__thread = None
552
+
553
+ if self.__loop is not None:
554
+ self.__loop.call_soon_threadsafe(self.__loop.stop)
555
+ self.__loop.close()
556
+ self.__loop = None
557
+
558
+ self._tasks.clear()
559
+ self._periodic_task.clear()
560
+
561
+ def in_same_thread(self):
562
+ return threading.get_ident() == self.thread.id
563
+
564
+ def create_periodic_task(
565
+ self,
566
+ coro: CoroutineFunc,
567
+ loop_when: Optional[Callable[[], bool]] = None,
568
+ interval: int = 2,
569
+ on_stop: Optional[CoroutineFunc] = None,
570
+ ) -> _PeriodicTask:
571
+ if loop_when is None:
572
+ loop_when = lambda: True
573
+
574
+ async def task_body():
575
+ while loop_when():
576
+ await coro()
577
+ await asyncio.sleep(interval)
578
+ if on_stop is not None:
579
+ await on_stop()
580
+
581
+ task = self.create_task(task_body())
582
+ periodic_task = _PeriodicTask(
583
+ task=task,
584
+ on_stop=on_stop,
585
+ )
586
+ self._periodic_task.append(periodic_task)
587
+ task.add_done_callback(lambda f: self._periodic_task.remove(periodic_task))
588
+ return periodic_task
589
+
590
+ def cancel_periodic_task(self, periodic_task: _PeriodicTask):
591
+ if periodic_task.task.done():
592
+ return
593
+ try:
594
+ periodic_task.task.cancel()
595
+ if periodic_task.on_stop is not None:
596
+ self.run(periodic_task.on_stop())
597
+ except Exception: # pragma: no cover # noqa
598
+ logger.exception("Exception occurs when cancel task.")
599
+
600
+
601
+ evloop = EventLoop()
602
+
603
+
604
+ def async_to_sync(func):
605
+ @functools.wraps(func)
606
+ def wrapper(*args, **kwargs):
607
+ coro = func(*args, **kwargs)
608
+ return evloop.run(coro)
609
+ return wrapper
610
+
611
+
612
+ class ErrorFuture: # pragma: no cover
613
+ __slots__ = ('_name_', '_err_')
614
+
615
+ def __init__(self, name, error):
616
+ self._name_ = name
617
+ self._err_: Exception = error
618
+
619
+ def __getattr__(self, item):
620
+ raise self._err_
621
+
622
+ def _get_msg(self):
623
+ err = self._err_
624
+ ref_chain = [self._name_]
625
+
626
+ while isinstance(err, BadFutureError):
627
+ ref_chain.append(err.obj._name_) # noqa
628
+ err = err.obj
629
+
630
+ ref_str = ' -> '.join(ref_chain)
631
+ err_repr = repr(err)
632
+
633
+ return f"\nReference Chain: {ref_str}\n" \
634
+ f"Error: {err_repr}"
635
+
636
+ def __repr__(self):
637
+ return repr(self._err_)
638
+
639
+
640
+ class _TaskProp:
641
+ def __init__(self):
642
+ self.task = None
643
+ if _LATER_THAN_3_10:
644
+ self.done = asyncio.Event()
645
+ else:
646
+ self.done = asyncio.Event(loop=evloop.loop)
647
+
648
+ @property
649
+ def result(self):
650
+ return self.task.result()
651
+
652
+
653
+ # noinspection PyPep8Naming
654
+ class future_property(Generic[ReturnT]):
655
+ def __init__(
656
+ self,
657
+ func: Optional[Callable[[Any], Coroutine[ReturnT, Any, ReturnT]]] = None,
658
+ on_demand: bool = False
659
+ ):
660
+ self.func = func
661
+ self.on_demand = on_demand
662
+ if func is not None:
663
+ self.__doc__ = func.__doc__
664
+ self._tasks: Dict[int, _TaskProp] = {}
665
+
666
+ def __call__(self, func: Callable[[Any], Coroutine[ReturnT, Any, ReturnT]]):
667
+ self.func = func
668
+ self.__doc__ = func.__doc__
669
+ return self
670
+
671
+ def __set_name__(self, owner, name):
672
+ self.attrname = name
673
+
674
+ # noinspection PyProtectedMember
675
+ def __get__(self: FP, instance: Optional[Any], owner=None) -> Union[FP, ReturnT]:
676
+ if instance is None:
677
+ return self
678
+
679
+ assert self.func is not None
680
+
681
+ if self.on_demand:
682
+ result = evloop.run(self.func(instance))
683
+ setattr(instance, self.attrname, result)
684
+ return result
685
+
686
+ if instance not in self._tasks:
687
+ try:
688
+ return instance.__dict__[self.attrname]
689
+ except KeyError:
690
+ raise RuntimeError(
691
+ f'Cannot resolve property {self.attrname!r} '
692
+ f'for {instance}') from None
693
+
694
+ task = self._tasks[instance]
695
+
696
+ if (
697
+ not task.done.is_set()
698
+ and evloop.in_same_thread()
699
+ ):
700
+ evloop.run(self.wait_for(instance))
701
+
702
+ setattr(instance, self.attrname, task.result)
703
+ self._tasks.pop(instance, None)
704
+ result = task.result
705
+ if not task.done.is_set():
706
+ task.done.set()
707
+ return result
708
+
709
+ def submit(self, instance):
710
+ if self.on_demand or instance in self._tasks:
711
+ return
712
+
713
+ self._tasks[instance] = t = _TaskProp()
714
+ t.task = task = evloop.create_task(self.func(instance))
715
+ task.die_in_peace = True
716
+ task.add_done_callback(functools.partial(self.set_task_done, instance))
717
+
718
+ async def wait_for(self, instance):
719
+ if self.attrname in instance.__dict__:
720
+ return instance.__dict__[self.attrname]
721
+
722
+ await instance.__future_guard__.wait()
723
+
724
+ if self.attrname in instance.__dict__: # NB: await后任务可能已经结束
725
+ return instance.__dict__[self.attrname]
726
+
727
+ task = self._tasks[instance]
728
+ await task.done.wait()
729
+ return task.result
730
+
731
+ def set_task_done(self, instance, future):
732
+ if instance not in self._tasks:
733
+ # __get__ called before
734
+ return
735
+ self._tasks[instance].done.set()
736
+
737
+ def reset(self, instance):
738
+ if self.attrname not in instance.__dict__:
739
+ return
740
+
741
+ self._tasks.pop(instance, None)
742
+ delattr(instance, self.attrname)
743
+ self.submit(instance)
744
+
745
+
746
+ class FuturePropertyMeta(type):
747
+ def __init__(cls, name, bases, namespace):
748
+ submit_calls = {}
749
+
750
+ call_super = any(isinstance(clz, FuturePropertyMeta) for clz in bases)
751
+
752
+ for _name, attr in cls.__dict__.items():
753
+ if not isinstance(attr, future_property):
754
+ continue
755
+ submit_calls[_name] = attr.submit
756
+
757
+ ori_init = cls.__init__
758
+
759
+ if ori_init is None:
760
+ def new_init(self, *_args, **_kwargs):
761
+ self.__fire_futures__(set())
762
+
763
+ else:
764
+ def new_init(self, *_args, **_kwargs):
765
+ ori_init(self, *_args, **_kwargs)
766
+ self.__fire_futures__(set())
767
+
768
+ cls.__init__ = new_init
769
+ cls.__future_fired__ = False
770
+ if callable(ori_init):
771
+ cls.__init__.__signature__ = inspect.signature(ori_init)
772
+
773
+ def __fire_futures__(self, skip: Set[str], is_root_call: bool = True):
774
+ if is_root_call and not hasattr(self, '__future_guard__'):
775
+ if evloop.in_same_thread():
776
+ self.__future_guard__ = asyncio.Event()
777
+ else:
778
+ async def attch_guard():
779
+ self.__future_guard__ = asyncio.Event()
780
+ evloop.run(attch_guard())
781
+
782
+ if self.__future_fired__:
783
+ return
784
+
785
+ if call_super:
786
+ # NB: 必须先调用super, 因为子类的future可能依赖父类,如果后调用super,可能导致
787
+ # 子类future运行时父类还未提交,触发task中的KeyError
788
+ super(cls, self).__fire_futures__(skip.union(submit_calls), False) # noqa
789
+
790
+ for _n, call in submit_calls.items():
791
+ if _n not in skip:
792
+ call(self) # noqa
793
+
794
+ if is_root_call:
795
+ self.__future_fired__ = True
796
+ evloop.loop.call_soon_threadsafe(self.__future_guard__.set)
797
+
798
+ cls.__fire_futures__ = __fire_futures__
799
+ super().__init__(name, bases, namespace)
800
+
801
+
802
+ def cache_async(coro_func):
803
+ lock_key = '__cache_lock__'
804
+
805
+ @functools.wraps(coro_func)
806
+ async def wrapper(self, *args, **kwargs):
807
+ key = f"_{coro_func.__name__}_"
808
+ if not hasattr(self, lock_key):
809
+ setattr(self, lock_key, asyncio.Lock())
810
+
811
+ async with getattr(self, lock_key):
812
+ cache = getattr(self, key, None)
813
+ if cache is not None:
814
+ return cache
815
+
816
+ value = await coro_func(self, *args, **kwargs)
817
+ setattr(self, key, value)
818
+ return value
819
+ return wrapper
820
+
821
+
822
+ @eliminate_from_traceback
823
+ class DeepFOSIO:
824
+ def __init__(self):
825
+ self.__running = False
826
+ self.on_startup: CallbacksT = []
827
+ self.on_shutdown: CallbacksT = []
828
+
829
+ def register_shutdown(self, coro: Callable, is_coro=UNSET):
830
+ if not self.__running:
831
+ return
832
+ self.on_shutdown.append((coro, is_coro))
833
+
834
+ def register_startup(self, coro: Callable, is_coro=UNSET):
835
+ if self.__running:
836
+ return
837
+ self.on_startup.append((coro, is_coro))
838
+
839
+ def run(self, coro: Coroutine):
840
+ self.__running = True
841
+ try:
842
+ return asyncio.run(self._managed_run(coro))
843
+ finally:
844
+ self.__running = False
845
+
846
+ @staticmethod
847
+ async def apply_handler(handlers: CallbacksT):
848
+ for handler, is_coro in handlers:
849
+ if is_coro is True:
850
+ await handler()
851
+ elif is_coro is False:
852
+ handler()
853
+ else:
854
+ if inspect.iscoroutinefunction(handler):
855
+ await handler()
856
+ else:
857
+ handler()
858
+ handlers.clear()
859
+
860
+ async def _managed_run(self, main: Coroutine):
861
+ await self.apply_handler(self.on_startup)
862
+ self.on_startup.clear()
863
+ try:
864
+ return await main
865
+ finally:
866
+ try:
867
+ await self.apply_handler(self.on_shutdown)
868
+ except Exception: # pragma: no cover # noqa
869
+ pass
870
+
871
+
872
+ deepfosio = DeepFOSIO()
873
+
874
+
875
+ def register_on_loop_shutdown(coro_or_func, is_coro=UNSET):
876
+ if evloop.in_same_thread():
877
+ evloop.register_shutdown(coro_or_func, is_coro)
878
+ else:
879
+ deepfosio.register_shutdown(coro_or_func, is_coro)