codie-util-twisted 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codie_util_twisted/__init__.py +1 -0
- codie_util_twisted/defer_util.py +142 -0
- codie_util_twisted/defer_util_test.py +86 -0
- codie_util_twisted/mypy_twisted_plugin.py +53 -0
- codie_util_twisted/py.typed +1 -0
- codie_util_twisted/typed_reactor.py +62 -0
- codie_util_twisted-0.1.0.dist-info/METADATA +14 -0
- codie_util_twisted-0.1.0.dist-info/RECORD +9 -0
- codie_util_twisted-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '0.1.0'
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import functools
|
|
3
|
+
import inspect
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import Any, ParamSpec, TypeVar, cast
|
|
7
|
+
|
|
8
|
+
import twisted.python.threadable
|
|
9
|
+
from twisted.internet.defer import Deferred
|
|
10
|
+
from twisted.internet.threads import deferToThread
|
|
11
|
+
|
|
12
|
+
from codie_util_twisted.typed_reactor import TypedReactor
|
|
13
|
+
|
|
14
|
+
_P = ParamSpec("_P")
|
|
15
|
+
_R = TypeVar("_R")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def convertAsyncioMethodToDeferred(
|
|
19
|
+
method: Callable[_P, Any],
|
|
20
|
+
) -> Callable[_P, Deferred[Any]]:
|
|
21
|
+
@functools.wraps(method)
|
|
22
|
+
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> Deferred[Any]:
|
|
23
|
+
return Deferred.fromFuture(asyncio.create_task(method(*args, **kwargs)))
|
|
24
|
+
|
|
25
|
+
return wrapper
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class YieldInDeferToThreadException(Exception):
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def vortexLogFailure(
|
|
33
|
+
failure: Any,
|
|
34
|
+
loggerArg: logging.Logger,
|
|
35
|
+
consumeError: bool = False,
|
|
36
|
+
successValue: Any = True,
|
|
37
|
+
) -> Any:
|
|
38
|
+
"""Log a Twisted Failure instance using the provided logger."""
|
|
39
|
+
try:
|
|
40
|
+
if not hasattr(failure, "_vortexFailureLogged"):
|
|
41
|
+
exc_info = (
|
|
42
|
+
failure.type,
|
|
43
|
+
failure.value,
|
|
44
|
+
failure.getTracebackObject(),
|
|
45
|
+
)
|
|
46
|
+
loggerArg.exception(str(failure.value), exc_info=exc_info)
|
|
47
|
+
failure._vortexFailureLogged = True
|
|
48
|
+
return successValue if consumeError else failure
|
|
49
|
+
except Exception as exc:
|
|
50
|
+
loggerArg.exception("Error while logging failure: %s", str(exc))
|
|
51
|
+
return failure
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def deferToThreadWrapWithLogger(
|
|
55
|
+
logger: logging.Logger,
|
|
56
|
+
consumeError: bool = False,
|
|
57
|
+
checkMainThread: bool = True,
|
|
58
|
+
) -> Callable[[Callable[_P, _R]], Callable[_P, Deferred[_R]]]:
|
|
59
|
+
"""Decorator to run a blocking function in a thread pool via deferToThread.
|
|
60
|
+
|
|
61
|
+
Usage::
|
|
62
|
+
|
|
63
|
+
import logging
|
|
64
|
+
logger = logging.getLogger(__name__)
|
|
65
|
+
|
|
66
|
+
@deferToThreadWrapWithLogger(logger)
|
|
67
|
+
def myBlockingFunction(arg1, kw=True):
|
|
68
|
+
pass
|
|
69
|
+
|
|
70
|
+
The decorated function returns a Deferred when called. Yield it inside
|
|
71
|
+
an @inlineCallbacks function.
|
|
72
|
+
"""
|
|
73
|
+
assert isinstance(logger, logging.Logger), (
|
|
74
|
+
"deferToThreadWrapWithLogger requires a logging.Logger from "
|
|
75
|
+
"logging.getLogger(__name__)"
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def wrapper(funcToWrap: Callable[_P, _R]) -> Callable[_P, Deferred[_R]]:
|
|
79
|
+
def checkForInlineCallbackYield(result: _R) -> _R:
|
|
80
|
+
if inspect.isgenerator(result):
|
|
81
|
+
logger.error(
|
|
82
|
+
"Module Name %s, Class Name %s, Function Name %s has a yield in it. IT SHOULD NOT.",
|
|
83
|
+
funcToWrap.__module__,
|
|
84
|
+
funcToWrap.__class__.__name__,
|
|
85
|
+
funcToWrap.__name__,
|
|
86
|
+
)
|
|
87
|
+
raise YieldInDeferToThreadException()
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
def func(*args: _P.args, **kwargs: _P.kwargs) -> Deferred[_R]:
|
|
91
|
+
if not twisted.python.threadable.isInIOThread() and checkMainThread:
|
|
92
|
+
logger.error(
|
|
93
|
+
"Module Name %s, Class Name %s, Function Name %s is being deferred from a thread. IT SHOULD NOT.",
|
|
94
|
+
funcToWrap.__module__,
|
|
95
|
+
funcToWrap.__class__.__name__,
|
|
96
|
+
funcToWrap.__name__,
|
|
97
|
+
)
|
|
98
|
+
raise Exception(
|
|
99
|
+
"Deferring to a thread can only be done from the main thread"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
deferred: Deferred[_R] = deferToThread(funcToWrap, *args, **kwargs)
|
|
103
|
+
deferred.addCallback(checkForInlineCallbackYield)
|
|
104
|
+
deferred.addErrback(
|
|
105
|
+
vortexLogFailure, logger, consumeError=consumeError
|
|
106
|
+
)
|
|
107
|
+
return deferred
|
|
108
|
+
|
|
109
|
+
return func
|
|
110
|
+
|
|
111
|
+
return wrapper
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def deferOntoReactor(callback: Callable[_P, _R]) -> Callable[_P, _R]:
|
|
115
|
+
"""Decorator to defer a callback onto the reactor via callLater(0, ...).
|
|
116
|
+
|
|
117
|
+
Usage::
|
|
118
|
+
|
|
119
|
+
@deferOntoReactor
|
|
120
|
+
def _logicOnTupleObservableUpdate(self, tuples: list) -> None:
|
|
121
|
+
...
|
|
122
|
+
|
|
123
|
+
The wrapped callable is scheduled with ``reactor.callLater(0, ...)``
|
|
124
|
+
instead of running inline at notify time, so tuple-observable
|
|
125
|
+
subscribe callbacks never execute re-entrantly inside the
|
|
126
|
+
observable's notification loop
|
|
127
|
+
(``coding_standards/py_tx_observable_calllater``).
|
|
128
|
+
|
|
129
|
+
The wrapper returns immediately with ``None``; the wrapped callable's
|
|
130
|
+
return value is discarded by the scheduled invocation, and the
|
|
131
|
+
``DelayedCall`` from ``callLater`` is not leaked to the caller.
|
|
132
|
+
|
|
133
|
+
:param callback: The callback to schedule onto the reactor.
|
|
134
|
+
:return: A wrapper that schedules ``callback`` via ``callLater(0, ...)``.
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
@functools.wraps(callback)
|
|
138
|
+
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
|
|
139
|
+
TypedReactor.get().callLater(0, callback, *args, **kwargs)
|
|
140
|
+
return cast(_R, None)
|
|
141
|
+
|
|
142
|
+
return wrapper
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Tests for the reactor-scheduling helpers in defer_util."""
|
|
2
|
+
|
|
3
|
+
import twisted.internet
|
|
4
|
+
from collections.abc import Callable
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from twisted.internet import task
|
|
9
|
+
|
|
10
|
+
from codie_util_twisted.defer_util import deferOntoReactor
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@pytest.fixture
|
|
14
|
+
def clock(monkeypatch: pytest.MonkeyPatch) -> task.Clock:
|
|
15
|
+
clock = task.Clock()
|
|
16
|
+
monkeypatch.setattr(twisted.internet, "reactor", clock)
|
|
17
|
+
return clock
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_deferOntoReactorDefersCallUntilReactorRuns(
|
|
21
|
+
clock: task.Clock,
|
|
22
|
+
) -> None:
|
|
23
|
+
calls: list[tuple[tuple[Any, ...], dict[str, Any]]] = []
|
|
24
|
+
|
|
25
|
+
@deferOntoReactor
|
|
26
|
+
def callback(*args: Any, **kwargs: Any) -> None:
|
|
27
|
+
calls.append((args, kwargs))
|
|
28
|
+
|
|
29
|
+
callback(1, key="value")
|
|
30
|
+
assert calls == []
|
|
31
|
+
|
|
32
|
+
clock.advance(0)
|
|
33
|
+
assert calls == [((1,), {"key": "value"})]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_deferOntoReactorWrapperReturnsNoneNotDelayedCall(
|
|
37
|
+
clock: task.Clock,
|
|
38
|
+
) -> None:
|
|
39
|
+
wrapped: Callable[[], Any] = deferOntoReactor(lambda: "value")
|
|
40
|
+
|
|
41
|
+
assert wrapped() is None
|
|
42
|
+
|
|
43
|
+
clock.advance(0)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_deferOntoReactorRunsAfterPriorScheduledCalls(
|
|
47
|
+
clock: task.Clock,
|
|
48
|
+
) -> None:
|
|
49
|
+
events: list[str] = []
|
|
50
|
+
|
|
51
|
+
@deferOntoReactor
|
|
52
|
+
def callback() -> None:
|
|
53
|
+
events.append("wrapped")
|
|
54
|
+
|
|
55
|
+
clock.callLater(0, events.append, "prior")
|
|
56
|
+
callback()
|
|
57
|
+
assert events == []
|
|
58
|
+
|
|
59
|
+
clock.advance(0)
|
|
60
|
+
assert events == ["prior", "wrapped"]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_deferOntoReactorPreservesInvocationOrder(
|
|
64
|
+
clock: task.Clock,
|
|
65
|
+
) -> None:
|
|
66
|
+
events: list[int] = []
|
|
67
|
+
|
|
68
|
+
@deferOntoReactor
|
|
69
|
+
def callback(value: int) -> None:
|
|
70
|
+
events.append(value)
|
|
71
|
+
|
|
72
|
+
callback(1)
|
|
73
|
+
callback(2)
|
|
74
|
+
assert events == []
|
|
75
|
+
|
|
76
|
+
clock.advance(0)
|
|
77
|
+
assert events == [1, 2]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_deferOntoReactorPreservesFunctionMetadata() -> None:
|
|
81
|
+
@deferOntoReactor
|
|
82
|
+
def myCallback() -> None:
|
|
83
|
+
"""Callback docstring."""
|
|
84
|
+
|
|
85
|
+
assert myCallback.__name__ == "myCallback"
|
|
86
|
+
assert myCallback.__doc__ == "Callback docstring."
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Mypy plugin: validate @defer.inlineCallbacks return type annotations.
|
|
2
|
+
|
|
3
|
+
Checks that the inner return type T in Generator[Any, Any, T] is not itself
|
|
4
|
+
a Generator or Deferred — which would indicate a double-wrapping bug.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from typing import Callable
|
|
8
|
+
from mypy.plugin import Plugin, FunctionContext
|
|
9
|
+
from mypy.types import Type, get_proper_type, Instance, AnyType
|
|
10
|
+
|
|
11
|
+
_INLINE_CALLBACKS = {
|
|
12
|
+
"twisted.internet.defer.inlineCallbacks",
|
|
13
|
+
}
|
|
14
|
+
_FORBIDDEN_INNER = {
|
|
15
|
+
"builtins.generator",
|
|
16
|
+
"typing.Generator",
|
|
17
|
+
"twisted.internet.defer.Deferred",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _check_inner_type(ctx: FunctionContext) -> Type:
|
|
22
|
+
ret = ctx.default_return_type
|
|
23
|
+
proper = get_proper_type(ret)
|
|
24
|
+
# ret is Callable[P, Deferred[T]] after the stub transforms it.
|
|
25
|
+
# We inspect the Deferred's type argument to validate T.
|
|
26
|
+
if not isinstance(proper, Instance):
|
|
27
|
+
return ret
|
|
28
|
+
if not proper.args:
|
|
29
|
+
return ret
|
|
30
|
+
# proper is Deferred[T]; proper.args[0] is T
|
|
31
|
+
inner = get_proper_type(proper.args[0])
|
|
32
|
+
if isinstance(inner, AnyType):
|
|
33
|
+
return ret
|
|
34
|
+
if isinstance(inner, Instance) and inner.type.fullname in _FORBIDDEN_INNER:
|
|
35
|
+
ctx.api.fail(
|
|
36
|
+
f"@inlineCallbacks return type must not be Generator or Deferred "
|
|
37
|
+
f"(got {inner.type.fullname}); use the inner value type directly",
|
|
38
|
+
ctx.context,
|
|
39
|
+
)
|
|
40
|
+
return ret
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TwistedPlugin(Plugin):
|
|
44
|
+
def get_function_decorator_hook(
|
|
45
|
+
self, fullname: str
|
|
46
|
+
) -> Callable[[FunctionContext], Type] | None:
|
|
47
|
+
if fullname in _INLINE_CALLBACKS:
|
|
48
|
+
return _check_inner_type
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def plugin(_version: str) -> type[TwistedPlugin]:
|
|
53
|
+
return TwistedPlugin
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
partial
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Typed accessor for the Twisted global reactor."""
|
|
2
|
+
|
|
3
|
+
from typing import Any, Protocol, cast
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class _Reactor(Protocol):
|
|
7
|
+
"""Subset of the Twisted reactor interface used across DigOps apps."""
|
|
8
|
+
|
|
9
|
+
def callLater(
|
|
10
|
+
self, _delay: float, _callable: Any, *args: Any, **kwargs: Any
|
|
11
|
+
) -> Any: ...
|
|
12
|
+
|
|
13
|
+
def callWhenRunning(
|
|
14
|
+
self, _callable: Any, *args: Any, **kwargs: Any
|
|
15
|
+
) -> Any: ...
|
|
16
|
+
|
|
17
|
+
def callFromThread(
|
|
18
|
+
self, _callable: Any, *args: Any, **kwargs: Any
|
|
19
|
+
) -> None: ...
|
|
20
|
+
|
|
21
|
+
def addSystemEventTrigger(
|
|
22
|
+
self,
|
|
23
|
+
_phase: str,
|
|
24
|
+
_eventType: str,
|
|
25
|
+
_callable: Any,
|
|
26
|
+
*args: Any,
|
|
27
|
+
**kwargs: Any,
|
|
28
|
+
) -> Any: ...
|
|
29
|
+
|
|
30
|
+
def iterate(self, _delay: float = 0) -> None: ...
|
|
31
|
+
|
|
32
|
+
def listenTCP(
|
|
33
|
+
self, _port: int, _factory: Any, *args: Any, **kwargs: Any
|
|
34
|
+
) -> Any: ...
|
|
35
|
+
|
|
36
|
+
def run(self, *args: Any, **kwargs: Any) -> None: ...
|
|
37
|
+
|
|
38
|
+
def stop(self) -> None: ...
|
|
39
|
+
|
|
40
|
+
def suggestThreadPoolSize(self, _size: int) -> None: ...
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def running(self) -> bool: ...
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class TypedReactor:
|
|
47
|
+
"""Class-based accessor for the installed Twisted reactor."""
|
|
48
|
+
|
|
49
|
+
DEFAULT_THREAD_POOL_MAX = 500
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def get(cls) -> _Reactor:
|
|
53
|
+
from twisted.internet import reactor
|
|
54
|
+
|
|
55
|
+
return cast(_Reactor, reactor)
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
def logicSuggestThreadPoolSize(
|
|
59
|
+
cls, maxSize: int = DEFAULT_THREAD_POOL_MAX
|
|
60
|
+
) -> None:
|
|
61
|
+
"""Raise Twisted's deferToThread pool cap (default is 10)."""
|
|
62
|
+
cls.get().suggestThreadPoolSize(maxSize)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: codie-util-twisted
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
License-Expression: MIT
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
Requires-Dist: mypy>=1.0
|
|
7
|
+
Requires-Dist: twisted<26,>=25.5.0
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
Requires-Dist: basedpyright>=1.20; extra == 'dev'
|
|
10
|
+
Requires-Dist: pytest-twisted>=1.14; extra == 'dev'
|
|
11
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
12
|
+
Requires-Dist: ruff>=0.15; extra == 'dev'
|
|
13
|
+
Requires-Dist: taskipy>=1.13; extra == 'dev'
|
|
14
|
+
Requires-Dist: vulture>=2.14; extra == 'dev'
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
codie_util_twisted/__init__.py,sha256=IMjkMO3twhQzluVTo8Z6rE7Eg-9U79_LGKMcsWLKBkY,22
|
|
2
|
+
codie_util_twisted/defer_util.py,sha256=kNCWNFg81j1hHRDmNt8_gdK6jWyEo_HvxQTpC8JY02Y,4813
|
|
3
|
+
codie_util_twisted/defer_util_test.py,sha256=SHsC0TSST5etlGc63dOynoIfRgIBfzTGW5OD6_tICEg,1950
|
|
4
|
+
codie_util_twisted/mypy_twisted_plugin.py,sha256=aH4k2W3sPquUCYUVc6QCPE-YfP8x3H8iXtmNxT37INM,1685
|
|
5
|
+
codie_util_twisted/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
6
|
+
codie_util_twisted/typed_reactor.py,sha256=I26rVXfuO6t26K2j_PHfvgwxoM1Yn7MUzh0ckc6Mzp8,1542
|
|
7
|
+
codie_util_twisted-0.1.0.dist-info/METADATA,sha256=tyAgua5tWQbPcUnJ71dlH0ONG6LZl2OVgfwzdRofiW8,467
|
|
8
|
+
codie_util_twisted-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
9
|
+
codie_util_twisted-0.1.0.dist-info/RECORD,,
|