ominfra 0.0.0.dev129__py3-none-any.whl → 0.0.0.dev130__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/deploy/_executor.py +15 -0
- ominfra/pyremote/_runcommands.py +15 -0
- ominfra/scripts/journald2aws.py +112 -0
- ominfra/scripts/supervisor.py +1233 -484
- ominfra/supervisor/dispatchers.py +7 -6
- ominfra/supervisor/dispatchersimpl.py +3 -7
- ominfra/supervisor/groups.py +16 -1
- ominfra/supervisor/http.py +130 -0
- ominfra/supervisor/inject.py +44 -5
- ominfra/supervisor/io.py +39 -24
- ominfra/supervisor/processimpl.py +3 -4
- ominfra/supervisor/spawningimpl.py +2 -2
- ominfra/supervisor/supervisor.py +5 -2
- ominfra/supervisor/types.py +6 -56
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/RECORD +20 -20
- ominfra/supervisor/poller.py +0 -240
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
5
|
# @omlish-amalg-output ../supervisor/main.py
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP012 UP036
|
6
|
+
# ruff: noqa: N802 U006 UP006 UP007 UP012 UP036
|
7
7
|
# Supervisor is licensed under the following license:
|
8
8
|
#
|
9
9
|
# A copyright notice accompanies this license document that identifies the copyright holders.
|
@@ -102,6 +102,9 @@ V = ta.TypeVar('V')
|
|
102
102
|
# ../../../omlish/lite/cached.py
|
103
103
|
T = ta.TypeVar('T')
|
104
104
|
|
105
|
+
# ../../../omlish/lite/check.py
|
106
|
+
SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
|
107
|
+
|
105
108
|
# ../../../omlish/lite/socket.py
|
106
109
|
SocketAddress = ta.Any
|
107
110
|
SocketHandlerFactory = ta.Callable[[SocketAddress, ta.BinaryIO, ta.BinaryIO], 'SocketHandler']
|
@@ -1523,6 +1526,228 @@ def check_single(vs: ta.Iterable[T]) -> T:
|
|
1523
1526
|
return v
|
1524
1527
|
|
1525
1528
|
|
1529
|
+
def check_empty(v: SizedT) -> SizedT:
|
1530
|
+
if len(v):
|
1531
|
+
raise ValueError(v)
|
1532
|
+
return v
|
1533
|
+
|
1534
|
+
|
1535
|
+
def check_non_empty(v: SizedT) -> SizedT:
|
1536
|
+
if not len(v):
|
1537
|
+
raise ValueError(v)
|
1538
|
+
return v
|
1539
|
+
|
1540
|
+
|
1541
|
+
########################################
|
1542
|
+
# ../../../omlish/lite/fdio/pollers.py
|
1543
|
+
|
1544
|
+
|
1545
|
+
##
|
1546
|
+
|
1547
|
+
|
1548
|
+
class FdIoPoller(abc.ABC):
|
1549
|
+
def __init__(self) -> None:
|
1550
|
+
super().__init__()
|
1551
|
+
|
1552
|
+
self._readable: ta.Set[int] = set()
|
1553
|
+
self._writable: ta.Set[int] = set()
|
1554
|
+
|
1555
|
+
#
|
1556
|
+
|
1557
|
+
def close(self) -> None: # noqa
|
1558
|
+
pass
|
1559
|
+
|
1560
|
+
def reopen(self) -> None: # noqa
|
1561
|
+
pass
|
1562
|
+
|
1563
|
+
#
|
1564
|
+
|
1565
|
+
@property
|
1566
|
+
@ta.final
|
1567
|
+
def readable(self) -> ta.AbstractSet[int]:
|
1568
|
+
return self._readable
|
1569
|
+
|
1570
|
+
@property
|
1571
|
+
@ta.final
|
1572
|
+
def writable(self) -> ta.AbstractSet[int]:
|
1573
|
+
return self._writable
|
1574
|
+
|
1575
|
+
#
|
1576
|
+
|
1577
|
+
@ta.final
|
1578
|
+
def register_readable(self, fd: int) -> bool:
|
1579
|
+
if fd in self._readable:
|
1580
|
+
return False
|
1581
|
+
self._readable.add(fd)
|
1582
|
+
self._register_readable(fd)
|
1583
|
+
return True
|
1584
|
+
|
1585
|
+
@ta.final
|
1586
|
+
def register_writable(self, fd: int) -> bool:
|
1587
|
+
if fd in self._writable:
|
1588
|
+
return False
|
1589
|
+
self._writable.add(fd)
|
1590
|
+
self._register_writable(fd)
|
1591
|
+
return True
|
1592
|
+
|
1593
|
+
@ta.final
|
1594
|
+
def unregister_readable(self, fd: int) -> bool:
|
1595
|
+
if fd not in self._readable:
|
1596
|
+
return False
|
1597
|
+
self._readable.discard(fd)
|
1598
|
+
self._unregister_readable(fd)
|
1599
|
+
return True
|
1600
|
+
|
1601
|
+
@ta.final
|
1602
|
+
def unregister_writable(self, fd: int) -> bool:
|
1603
|
+
if fd not in self._writable:
|
1604
|
+
return False
|
1605
|
+
self._writable.discard(fd)
|
1606
|
+
self._unregister_writable(fd)
|
1607
|
+
return True
|
1608
|
+
|
1609
|
+
#
|
1610
|
+
|
1611
|
+
def _register_readable(self, fd: int) -> None: # noqa
|
1612
|
+
pass
|
1613
|
+
|
1614
|
+
def _register_writable(self, fd: int) -> None: # noqa
|
1615
|
+
pass
|
1616
|
+
|
1617
|
+
def _unregister_readable(self, fd: int) -> None: # noqa
|
1618
|
+
pass
|
1619
|
+
|
1620
|
+
def _unregister_writable(self, fd: int) -> None: # noqa
|
1621
|
+
pass
|
1622
|
+
|
1623
|
+
#
|
1624
|
+
|
1625
|
+
def update(
|
1626
|
+
self,
|
1627
|
+
r: ta.AbstractSet[int],
|
1628
|
+
w: ta.AbstractSet[int],
|
1629
|
+
) -> None:
|
1630
|
+
for f in r - self._readable:
|
1631
|
+
self.register_readable(f)
|
1632
|
+
for f in w - self._writable:
|
1633
|
+
self.register_writable(f)
|
1634
|
+
for f in self._readable - r:
|
1635
|
+
self.unregister_readable(f)
|
1636
|
+
for f in self._writable - w:
|
1637
|
+
self.unregister_writable(f)
|
1638
|
+
|
1639
|
+
#
|
1640
|
+
|
1641
|
+
@dc.dataclass(frozen=True)
|
1642
|
+
class PollResult:
|
1643
|
+
r: ta.Sequence[int] = ()
|
1644
|
+
w: ta.Sequence[int] = ()
|
1645
|
+
|
1646
|
+
inv: ta.Sequence[int] = ()
|
1647
|
+
|
1648
|
+
msg: ta.Optional[str] = None
|
1649
|
+
exc: ta.Optional[BaseException] = None
|
1650
|
+
|
1651
|
+
@abc.abstractmethod
|
1652
|
+
def poll(self, timeout: ta.Optional[float]) -> PollResult:
|
1653
|
+
raise NotImplementedError
|
1654
|
+
|
1655
|
+
|
1656
|
+
##
|
1657
|
+
|
1658
|
+
|
1659
|
+
class SelectFdIoPoller(FdIoPoller):
|
1660
|
+
def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
|
1661
|
+
try:
|
1662
|
+
r, w, x = select.select(
|
1663
|
+
self._readable,
|
1664
|
+
self._writable,
|
1665
|
+
[],
|
1666
|
+
timeout,
|
1667
|
+
)
|
1668
|
+
|
1669
|
+
except OSError as exc:
|
1670
|
+
if exc.errno == errno.EINTR:
|
1671
|
+
return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
|
1672
|
+
elif exc.errno == errno.EBADF:
|
1673
|
+
return FdIoPoller.PollResult(msg='EBADF encountered in poll', exc=exc)
|
1674
|
+
else:
|
1675
|
+
raise
|
1676
|
+
|
1677
|
+
return FdIoPoller.PollResult(r, w)
|
1678
|
+
|
1679
|
+
|
1680
|
+
##
|
1681
|
+
|
1682
|
+
|
1683
|
+
PollFdIoPoller: ta.Optional[ta.Type[FdIoPoller]]
|
1684
|
+
if hasattr(select, 'poll'):
|
1685
|
+
|
1686
|
+
class _PollFdIoPoller(FdIoPoller):
|
1687
|
+
def __init__(self) -> None:
|
1688
|
+
super().__init__()
|
1689
|
+
|
1690
|
+
self._poller = select.poll()
|
1691
|
+
|
1692
|
+
#
|
1693
|
+
|
1694
|
+
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
1695
|
+
_WRITE = select.POLLOUT
|
1696
|
+
|
1697
|
+
def _register_readable(self, fd: int) -> None:
|
1698
|
+
self._update_registration(fd)
|
1699
|
+
|
1700
|
+
def _register_writable(self, fd: int) -> None:
|
1701
|
+
self._update_registration(fd)
|
1702
|
+
|
1703
|
+
def _unregister_readable(self, fd: int) -> None:
|
1704
|
+
self._update_registration(fd)
|
1705
|
+
|
1706
|
+
def _unregister_writable(self, fd: int) -> None:
|
1707
|
+
self._update_registration(fd)
|
1708
|
+
|
1709
|
+
def _update_registration(self, fd: int) -> None:
|
1710
|
+
r = fd in self._readable
|
1711
|
+
w = fd in self._writable
|
1712
|
+
if r or w:
|
1713
|
+
self._poller.register(fd, (self._READ if r else 0) | (self._WRITE if w else 0))
|
1714
|
+
else:
|
1715
|
+
self._poller.unregister(fd)
|
1716
|
+
|
1717
|
+
#
|
1718
|
+
|
1719
|
+
def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
|
1720
|
+
polled: ta.List[ta.Tuple[int, int]]
|
1721
|
+
try:
|
1722
|
+
polled = self._poller.poll(timeout * 1000 if timeout is not None else None)
|
1723
|
+
|
1724
|
+
except OSError as exc:
|
1725
|
+
if exc.errno == errno.EINTR:
|
1726
|
+
return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
|
1727
|
+
else:
|
1728
|
+
raise
|
1729
|
+
|
1730
|
+
r: ta.List[int] = []
|
1731
|
+
w: ta.List[int] = []
|
1732
|
+
inv: ta.List[int] = []
|
1733
|
+
for fd, mask in polled:
|
1734
|
+
if mask & select.POLLNVAL:
|
1735
|
+
self._poller.unregister(fd)
|
1736
|
+
self._readable.discard(fd)
|
1737
|
+
self._writable.discard(fd)
|
1738
|
+
inv.append(fd)
|
1739
|
+
continue
|
1740
|
+
if mask & self._READ:
|
1741
|
+
r.append(fd)
|
1742
|
+
if mask & self._WRITE:
|
1743
|
+
w.append(fd)
|
1744
|
+
return FdIoPoller.PollResult(r, w, inv=inv)
|
1745
|
+
|
1746
|
+
PollFdIoPoller = _PollFdIoPoller
|
1747
|
+
else:
|
1748
|
+
PollFdIoPoller = None
|
1749
|
+
|
1750
|
+
|
1526
1751
|
########################################
|
1527
1752
|
# ../../../omlish/lite/http/versions.py
|
1528
1753
|
|
@@ -1742,6 +1967,73 @@ class SocketHandler(abc.ABC):
|
|
1742
1967
|
raise NotImplementedError
|
1743
1968
|
|
1744
1969
|
|
1970
|
+
########################################
|
1971
|
+
# ../../../omlish/lite/strings.py
|
1972
|
+
|
1973
|
+
|
1974
|
+
##
|
1975
|
+
|
1976
|
+
|
1977
|
+
def camel_case(name: str, lower: bool = False) -> str:
|
1978
|
+
if not name:
|
1979
|
+
return ''
|
1980
|
+
s = ''.join(map(str.capitalize, name.split('_'))) # noqa
|
1981
|
+
if lower:
|
1982
|
+
s = s[0].lower() + s[1:]
|
1983
|
+
return s
|
1984
|
+
|
1985
|
+
|
1986
|
+
def snake_case(name: str) -> str:
|
1987
|
+
uppers: list[int | None] = [i for i, c in enumerate(name) if c.isupper()]
|
1988
|
+
return '_'.join([name[l:r].lower() for l, r in zip([None, *uppers], [*uppers, None])]).strip('_')
|
1989
|
+
|
1990
|
+
|
1991
|
+
##
|
1992
|
+
|
1993
|
+
|
1994
|
+
def is_dunder(name: str) -> bool:
|
1995
|
+
return (
|
1996
|
+
name[:2] == name[-2:] == '__' and
|
1997
|
+
name[2:3] != '_' and
|
1998
|
+
name[-3:-2] != '_' and
|
1999
|
+
len(name) > 4
|
2000
|
+
)
|
2001
|
+
|
2002
|
+
|
2003
|
+
def is_sunder(name: str) -> bool:
|
2004
|
+
return (
|
2005
|
+
name[0] == name[-1] == '_' and
|
2006
|
+
name[1:2] != '_' and
|
2007
|
+
name[-2:-1] != '_' and
|
2008
|
+
len(name) > 2
|
2009
|
+
)
|
2010
|
+
|
2011
|
+
|
2012
|
+
##
|
2013
|
+
|
2014
|
+
|
2015
|
+
def attr_repr(obj: ta.Any, *attrs: str) -> str:
|
2016
|
+
return f'{type(obj).__name__}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
|
2017
|
+
|
2018
|
+
|
2019
|
+
##
|
2020
|
+
|
2021
|
+
|
2022
|
+
FORMAT_NUM_BYTES_SUFFIXES: ta.Sequence[str] = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB']
|
2023
|
+
|
2024
|
+
|
2025
|
+
def format_num_bytes(num_bytes: int) -> str:
|
2026
|
+
for i, suffix in enumerate(FORMAT_NUM_BYTES_SUFFIXES):
|
2027
|
+
value = num_bytes / 1024 ** i
|
2028
|
+
if num_bytes < 1024 ** (i + 1):
|
2029
|
+
if value.is_integer():
|
2030
|
+
return f'{int(value)}{suffix}'
|
2031
|
+
else:
|
2032
|
+
return f'{value:.2f}{suffix}'
|
2033
|
+
|
2034
|
+
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
2035
|
+
|
2036
|
+
|
1745
2037
|
########################################
|
1746
2038
|
# ../../../omlish/lite/typing.py
|
1747
2039
|
|
@@ -2226,6 +2518,169 @@ def get_user(name: str) -> User:
|
|
2226
2518
|
)
|
2227
2519
|
|
2228
2520
|
|
2521
|
+
########################################
|
2522
|
+
# ../../../omlish/lite/fdio/handlers.py
|
2523
|
+
|
2524
|
+
|
2525
|
+
class FdIoHandler(abc.ABC):
|
2526
|
+
@abc.abstractmethod
|
2527
|
+
def fd(self) -> int:
|
2528
|
+
raise NotImplementedError
|
2529
|
+
|
2530
|
+
#
|
2531
|
+
|
2532
|
+
@property
|
2533
|
+
@abc.abstractmethod
|
2534
|
+
def closed(self) -> bool:
|
2535
|
+
raise NotImplementedError
|
2536
|
+
|
2537
|
+
@abc.abstractmethod
|
2538
|
+
def close(self) -> None:
|
2539
|
+
raise NotImplementedError
|
2540
|
+
|
2541
|
+
#
|
2542
|
+
|
2543
|
+
def readable(self) -> bool:
|
2544
|
+
return False
|
2545
|
+
|
2546
|
+
def writable(self) -> bool:
|
2547
|
+
return False
|
2548
|
+
|
2549
|
+
#
|
2550
|
+
|
2551
|
+
def on_readable(self) -> None:
|
2552
|
+
raise TypeError
|
2553
|
+
|
2554
|
+
def on_writable(self) -> None:
|
2555
|
+
raise TypeError
|
2556
|
+
|
2557
|
+
def on_error(self, exc: ta.Optional[BaseException] = None) -> None: # noqa
|
2558
|
+
pass
|
2559
|
+
|
2560
|
+
|
2561
|
+
class SocketFdIoHandler(FdIoHandler, abc.ABC):
|
2562
|
+
def __init__(
|
2563
|
+
self,
|
2564
|
+
addr: SocketAddress,
|
2565
|
+
sock: socket.socket,
|
2566
|
+
) -> None:
|
2567
|
+
super().__init__()
|
2568
|
+
|
2569
|
+
self._addr = addr
|
2570
|
+
self._sock: ta.Optional[socket.socket] = sock
|
2571
|
+
|
2572
|
+
def fd(self) -> int:
|
2573
|
+
return check_not_none(self._sock).fileno()
|
2574
|
+
|
2575
|
+
@property
|
2576
|
+
def closed(self) -> bool:
|
2577
|
+
return self._sock is None
|
2578
|
+
|
2579
|
+
def close(self) -> None:
|
2580
|
+
if self._sock is not None:
|
2581
|
+
self._sock.close()
|
2582
|
+
self._sock = None
|
2583
|
+
|
2584
|
+
|
2585
|
+
########################################
|
2586
|
+
# ../../../omlish/lite/fdio/kqueue.py
|
2587
|
+
|
2588
|
+
|
2589
|
+
KqueueFdIoPoller: ta.Optional[ta.Type[FdIoPoller]]
|
2590
|
+
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
2591
|
+
|
2592
|
+
class _KqueueFdIoPoller(FdIoPoller):
|
2593
|
+
DEFAULT_MAX_EVENTS = 1000
|
2594
|
+
|
2595
|
+
def __init__(
|
2596
|
+
self,
|
2597
|
+
*,
|
2598
|
+
max_events: int = DEFAULT_MAX_EVENTS,
|
2599
|
+
) -> None:
|
2600
|
+
super().__init__()
|
2601
|
+
|
2602
|
+
self._max_events = max_events
|
2603
|
+
|
2604
|
+
self._kqueue: ta.Optional[ta.Any] = None
|
2605
|
+
|
2606
|
+
#
|
2607
|
+
|
2608
|
+
def _get_kqueue(self) -> 'select.kqueue':
|
2609
|
+
if (kq := self._kqueue) is not None:
|
2610
|
+
return kq
|
2611
|
+
kq = select.kqueue()
|
2612
|
+
self._kqueue = kq
|
2613
|
+
return kq
|
2614
|
+
|
2615
|
+
def close(self) -> None:
|
2616
|
+
if self._kqueue is not None:
|
2617
|
+
self._kqueue.close()
|
2618
|
+
self._kqueue = None
|
2619
|
+
|
2620
|
+
def reopen(self) -> None:
|
2621
|
+
for fd in self._readable:
|
2622
|
+
self._register_readable(fd)
|
2623
|
+
for fd in self._writable:
|
2624
|
+
self._register_writable(fd)
|
2625
|
+
|
2626
|
+
#
|
2627
|
+
|
2628
|
+
def _register_readable(self, fd: int) -> None:
|
2629
|
+
self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_ADD)
|
2630
|
+
|
2631
|
+
def _register_writable(self, fd: int) -> None:
|
2632
|
+
self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)
|
2633
|
+
|
2634
|
+
def _unregister_readable(self, fd: int) -> None:
|
2635
|
+
self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE)
|
2636
|
+
|
2637
|
+
def _unregister_writable(self, fd: int) -> None:
|
2638
|
+
self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)
|
2639
|
+
|
2640
|
+
def _control(self, fd: int, filter: int, flags: int) -> None: # noqa
|
2641
|
+
ke = select.kevent(fd, filter=filter, flags=flags)
|
2642
|
+
kq = self._get_kqueue()
|
2643
|
+
try:
|
2644
|
+
kq.control([ke], 0)
|
2645
|
+
|
2646
|
+
except OSError as exc:
|
2647
|
+
if exc.errno == errno.EBADF:
|
2648
|
+
# log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', ke.ident)
|
2649
|
+
pass
|
2650
|
+
elif exc.errno == errno.ENOENT:
|
2651
|
+
# Can happen when trying to remove an already closed socket
|
2652
|
+
pass
|
2653
|
+
else:
|
2654
|
+
raise
|
2655
|
+
|
2656
|
+
#
|
2657
|
+
|
2658
|
+
def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
|
2659
|
+
kq = self._get_kqueue()
|
2660
|
+
try:
|
2661
|
+
kes = kq.control(None, self._max_events, timeout)
|
2662
|
+
|
2663
|
+
except OSError as exc:
|
2664
|
+
if exc.errno == errno.EINTR:
|
2665
|
+
return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
|
2666
|
+
else:
|
2667
|
+
raise
|
2668
|
+
|
2669
|
+
r: ta.List[int] = []
|
2670
|
+
w: ta.List[int] = []
|
2671
|
+
for ke in kes:
|
2672
|
+
if ke.filter == select.KQ_FILTER_READ:
|
2673
|
+
r.append(ke.ident)
|
2674
|
+
if ke.filter == select.KQ_FILTER_WRITE:
|
2675
|
+
w.append(ke.ident)
|
2676
|
+
|
2677
|
+
return FdIoPoller.PollResult(r, w)
|
2678
|
+
|
2679
|
+
KqueueFdIoPoller = _KqueueFdIoPoller
|
2680
|
+
else:
|
2681
|
+
KqueueFdIoPoller = None
|
2682
|
+
|
2683
|
+
|
2229
2684
|
########################################
|
2230
2685
|
# ../../../omlish/lite/http/parsing.py
|
2231
2686
|
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
@@ -2628,11 +3083,6 @@ class HttpRequestParser:
|
|
2628
3083
|
|
2629
3084
|
########################################
|
2630
3085
|
# ../../../omlish/lite/inject.py
|
2631
|
-
"""
|
2632
|
-
TODO:
|
2633
|
-
- recursion detection
|
2634
|
-
- bind empty array
|
2635
|
-
"""
|
2636
3086
|
|
2637
3087
|
|
2638
3088
|
###
|
@@ -2925,7 +3375,11 @@ def build_injector_provider_map(bs: InjectorBindings) -> ta.Mapping[InjectorKey,
|
|
2925
3375
|
|
2926
3376
|
for b in bs.bindings():
|
2927
3377
|
if b.key.array:
|
2928
|
-
am.setdefault(b.key, [])
|
3378
|
+
al = am.setdefault(b.key, [])
|
3379
|
+
if isinstance(b.provider, ArrayInjectorProvider):
|
3380
|
+
al.extend(b.provider.ps)
|
3381
|
+
else:
|
3382
|
+
al.append(b.provider)
|
2929
3383
|
else:
|
2930
3384
|
if b.key in pm:
|
2931
3385
|
raise KeyError(b.key)
|
@@ -3073,8 +3527,16 @@ def build_injection_kwargs_target(
|
|
3073
3527
|
_INJECTOR_INJECTOR_KEY: InjectorKey[Injector] = InjectorKey(Injector)
|
3074
3528
|
|
3075
3529
|
|
3076
|
-
|
3077
|
-
|
3530
|
+
@dc.dataclass(frozen=True)
|
3531
|
+
class _InjectorEager:
|
3532
|
+
key: InjectorKey
|
3533
|
+
|
3534
|
+
|
3535
|
+
_INJECTOR_EAGER_ARRAY_KEY: InjectorKey[_InjectorEager] = InjectorKey(_InjectorEager, array=True)
|
3536
|
+
|
3537
|
+
|
3538
|
+
class _Injector(Injector):
|
3539
|
+
def __init__(self, bs: InjectorBindings, p: ta.Optional[Injector] = None) -> None:
|
3078
3540
|
super().__init__()
|
3079
3541
|
|
3080
3542
|
self._bs = check_isinstance(bs, InjectorBindings)
|
@@ -3087,6 +3549,10 @@ class _Injector(Injector):
|
|
3087
3549
|
|
3088
3550
|
self.__cur_req: ta.Optional[_Injector._Request] = None
|
3089
3551
|
|
3552
|
+
if _INJECTOR_EAGER_ARRAY_KEY in self._pfm:
|
3553
|
+
for e in self.provide(_INJECTOR_EAGER_ARRAY_KEY):
|
3554
|
+
self.provide(e.key)
|
3555
|
+
|
3090
3556
|
class _Request:
|
3091
3557
|
def __init__(self, injector: '_Injector') -> None:
|
3092
3558
|
super().__init__()
|
@@ -3244,6 +3710,8 @@ class InjectorBinder:
|
|
3244
3710
|
to_key: ta.Any = None,
|
3245
3711
|
|
3246
3712
|
singleton: bool = False,
|
3713
|
+
|
3714
|
+
eager: bool = False,
|
3247
3715
|
) -> InjectorBindingOrBindings:
|
3248
3716
|
if obj is None or obj is inspect.Parameter.empty:
|
3249
3717
|
raise TypeError(obj)
|
@@ -3317,13 +3785,21 @@ class InjectorBinder:
|
|
3317
3785
|
if singleton:
|
3318
3786
|
provider = SingletonInjectorProvider(provider)
|
3319
3787
|
|
3788
|
+
binding = InjectorBinding(key, provider)
|
3789
|
+
|
3320
3790
|
##
|
3321
3791
|
|
3322
|
-
|
3792
|
+
extras: ta.List[InjectorBinding] = []
|
3793
|
+
|
3794
|
+
if eager:
|
3795
|
+
extras.append(bind_injector_eager_key(key))
|
3323
3796
|
|
3324
3797
|
##
|
3325
3798
|
|
3326
|
-
|
3799
|
+
if extras:
|
3800
|
+
return as_injector_bindings(binding, *extras)
|
3801
|
+
else:
|
3802
|
+
return binding
|
3327
3803
|
|
3328
3804
|
|
3329
3805
|
###
|
@@ -3346,6 +3822,26 @@ def make_injector_factory(
|
|
3346
3822
|
return outer
|
3347
3823
|
|
3348
3824
|
|
3825
|
+
def bind_injector_array(
|
3826
|
+
obj: ta.Any = None,
|
3827
|
+
*,
|
3828
|
+
tag: ta.Any = None,
|
3829
|
+
) -> InjectorBindingOrBindings:
|
3830
|
+
key = as_injector_key(obj)
|
3831
|
+
if tag is not None:
|
3832
|
+
if key.tag is not None:
|
3833
|
+
raise ValueError('Must not specify multiple tags')
|
3834
|
+
key = dc.replace(key, tag=tag)
|
3835
|
+
|
3836
|
+
if key.array:
|
3837
|
+
raise ValueError('Key must not be array')
|
3838
|
+
|
3839
|
+
return InjectorBinding(
|
3840
|
+
dc.replace(key, array=True),
|
3841
|
+
ArrayInjectorProvider([]),
|
3842
|
+
)
|
3843
|
+
|
3844
|
+
|
3349
3845
|
def make_injector_array_type(
|
3350
3846
|
ele: ta.Union[InjectorKey, InjectorKeyCls],
|
3351
3847
|
cls: U,
|
@@ -3367,6 +3863,10 @@ def make_injector_array_type(
|
|
3367
3863
|
return inner
|
3368
3864
|
|
3369
3865
|
|
3866
|
+
def bind_injector_eager_key(key: ta.Any) -> InjectorBinding:
|
3867
|
+
return InjectorBinding(_INJECTOR_EAGER_ARRAY_KEY, ConstInjectorProvider(_InjectorEager(as_injector_key(key))))
|
3868
|
+
|
3869
|
+
|
3370
3870
|
##
|
3371
3871
|
|
3372
3872
|
|
@@ -3421,6 +3921,8 @@ class Injection:
|
|
3421
3921
|
to_key: ta.Any = None,
|
3422
3922
|
|
3423
3923
|
singleton: bool = False,
|
3924
|
+
|
3925
|
+
eager: bool = False,
|
3424
3926
|
) -> InjectorBindingOrBindings:
|
3425
3927
|
return InjectorBinder.bind(
|
3426
3928
|
obj,
|
@@ -3435,6 +3937,8 @@ class Injection:
|
|
3435
3937
|
to_key=to_key,
|
3436
3938
|
|
3437
3939
|
singleton=singleton,
|
3940
|
+
|
3941
|
+
eager=eager,
|
3438
3942
|
)
|
3439
3943
|
|
3440
3944
|
# helpers
|
@@ -3448,6 +3952,15 @@ class Injection:
|
|
3448
3952
|
) -> InjectorBindingOrBindings:
|
3449
3953
|
return cls.bind(make_injector_factory(fn, cls_, ann))
|
3450
3954
|
|
3955
|
+
@classmethod
|
3956
|
+
def bind_array(
|
3957
|
+
cls,
|
3958
|
+
obj: ta.Any = None,
|
3959
|
+
*,
|
3960
|
+
tag: ta.Any = None,
|
3961
|
+
) -> InjectorBindingOrBindings:
|
3962
|
+
return bind_injector_array(obj, tag=tag)
|
3963
|
+
|
3451
3964
|
@classmethod
|
3452
3965
|
def bind_array_type(
|
3453
3966
|
cls,
|
@@ -3461,6 +3974,228 @@ class Injection:
|
|
3461
3974
|
inj = Injection
|
3462
3975
|
|
3463
3976
|
|
3977
|
+
########################################
|
3978
|
+
# ../../../omlish/lite/io.py
|
3979
|
+
|
3980
|
+
|
3981
|
+
class DelimitingBuffer:
|
3982
|
+
"""
|
3983
|
+
https://github.com/python-trio/trio/issues/796 :|
|
3984
|
+
"""
|
3985
|
+
|
3986
|
+
#
|
3987
|
+
|
3988
|
+
class Error(Exception):
|
3989
|
+
def __init__(self, buffer: 'DelimitingBuffer') -> None:
|
3990
|
+
super().__init__(buffer)
|
3991
|
+
self.buffer = buffer
|
3992
|
+
|
3993
|
+
def __repr__(self) -> str:
|
3994
|
+
return attr_repr(self, 'buffer')
|
3995
|
+
|
3996
|
+
class ClosedError(Error):
|
3997
|
+
pass
|
3998
|
+
|
3999
|
+
#
|
4000
|
+
|
4001
|
+
DEFAULT_DELIMITERS: bytes = b'\n'
|
4002
|
+
|
4003
|
+
def __init__(
|
4004
|
+
self,
|
4005
|
+
delimiters: ta.Iterable[int] = DEFAULT_DELIMITERS,
|
4006
|
+
*,
|
4007
|
+
keep_ends: bool = False,
|
4008
|
+
max_size: ta.Optional[int] = None,
|
4009
|
+
) -> None:
|
4010
|
+
super().__init__()
|
4011
|
+
|
4012
|
+
self._delimiters = frozenset(check_isinstance(d, int) for d in delimiters)
|
4013
|
+
self._keep_ends = keep_ends
|
4014
|
+
self._max_size = max_size
|
4015
|
+
|
4016
|
+
self._buf: ta.Optional[io.BytesIO] = io.BytesIO()
|
4017
|
+
|
4018
|
+
#
|
4019
|
+
|
4020
|
+
@property
|
4021
|
+
def is_closed(self) -> bool:
|
4022
|
+
return self._buf is None
|
4023
|
+
|
4024
|
+
def tell(self) -> int:
|
4025
|
+
if (buf := self._buf) is None:
|
4026
|
+
raise self.ClosedError(self)
|
4027
|
+
return buf.tell()
|
4028
|
+
|
4029
|
+
def peek(self) -> bytes:
|
4030
|
+
if (buf := self._buf) is None:
|
4031
|
+
raise self.ClosedError(self)
|
4032
|
+
return buf.getvalue()
|
4033
|
+
|
4034
|
+
def _find_delim(self, data: ta.Union[bytes, bytearray], i: int) -> ta.Optional[int]:
|
4035
|
+
r = None # type: int | None
|
4036
|
+
for d in self._delimiters:
|
4037
|
+
if (p := data.find(d, i)) >= 0:
|
4038
|
+
if r is None or p < r:
|
4039
|
+
r = p
|
4040
|
+
return r
|
4041
|
+
|
4042
|
+
def _append_and_reset(self, chunk: bytes) -> bytes:
|
4043
|
+
buf = check_not_none(self._buf)
|
4044
|
+
if not buf.tell():
|
4045
|
+
return chunk
|
4046
|
+
|
4047
|
+
buf.write(chunk)
|
4048
|
+
ret = buf.getvalue()
|
4049
|
+
buf.seek(0)
|
4050
|
+
buf.truncate()
|
4051
|
+
return ret
|
4052
|
+
|
4053
|
+
class Incomplete(ta.NamedTuple):
|
4054
|
+
b: bytes
|
4055
|
+
|
4056
|
+
def feed(self, data: ta.Union[bytes, bytearray]) -> ta.Generator[ta.Union[bytes, Incomplete], None, None]:
|
4057
|
+
if (buf := self._buf) is None:
|
4058
|
+
raise self.ClosedError(self)
|
4059
|
+
|
4060
|
+
if not data:
|
4061
|
+
self._buf = None
|
4062
|
+
|
4063
|
+
if buf.tell():
|
4064
|
+
yield self.Incomplete(buf.getvalue())
|
4065
|
+
|
4066
|
+
return
|
4067
|
+
|
4068
|
+
l = len(data)
|
4069
|
+
i = 0
|
4070
|
+
while i < l:
|
4071
|
+
if (p := self._find_delim(data, i)) is None:
|
4072
|
+
break
|
4073
|
+
|
4074
|
+
n = p + 1
|
4075
|
+
if self._keep_ends:
|
4076
|
+
p = n
|
4077
|
+
|
4078
|
+
yield self._append_and_reset(data[i:p])
|
4079
|
+
|
4080
|
+
i = n
|
4081
|
+
|
4082
|
+
if i >= l:
|
4083
|
+
return
|
4084
|
+
|
4085
|
+
if self._max_size is None:
|
4086
|
+
buf.write(data[i:])
|
4087
|
+
return
|
4088
|
+
|
4089
|
+
while i < l:
|
4090
|
+
remaining_data_len = l - i
|
4091
|
+
remaining_buf_capacity = self._max_size - buf.tell()
|
4092
|
+
|
4093
|
+
if remaining_data_len < remaining_buf_capacity:
|
4094
|
+
buf.write(data[i:])
|
4095
|
+
return
|
4096
|
+
|
4097
|
+
p = i + remaining_buf_capacity
|
4098
|
+
yield self.Incomplete(self._append_and_reset(data[i:p]))
|
4099
|
+
i = p
|
4100
|
+
|
4101
|
+
|
4102
|
+
class ReadableListBuffer:
|
4103
|
+
def __init__(self) -> None:
|
4104
|
+
super().__init__()
|
4105
|
+
self._lst: list[bytes] = []
|
4106
|
+
|
4107
|
+
def feed(self, d: bytes) -> None:
|
4108
|
+
if d:
|
4109
|
+
self._lst.append(d)
|
4110
|
+
|
4111
|
+
def _chop(self, i: int, e: int) -> bytes:
|
4112
|
+
lst = self._lst
|
4113
|
+
d = lst[i]
|
4114
|
+
|
4115
|
+
o = b''.join([
|
4116
|
+
*lst[:i],
|
4117
|
+
d[:e],
|
4118
|
+
])
|
4119
|
+
|
4120
|
+
self._lst = [
|
4121
|
+
*([d[e:]] if e < len(d) else []),
|
4122
|
+
*lst[i + 1:],
|
4123
|
+
]
|
4124
|
+
|
4125
|
+
return o
|
4126
|
+
|
4127
|
+
def read(self, n: ta.Optional[int] = None) -> ta.Optional[bytes]:
|
4128
|
+
if n is None:
|
4129
|
+
o = b''.join(self._lst)
|
4130
|
+
self._lst = []
|
4131
|
+
return o
|
4132
|
+
|
4133
|
+
if not (lst := self._lst):
|
4134
|
+
return None
|
4135
|
+
|
4136
|
+
c = 0
|
4137
|
+
for i, d in enumerate(lst):
|
4138
|
+
r = n - c
|
4139
|
+
if (l := len(d)) >= r:
|
4140
|
+
return self._chop(i, r)
|
4141
|
+
c += l
|
4142
|
+
|
4143
|
+
return None
|
4144
|
+
|
4145
|
+
def read_until(self, delim: bytes = b'\n') -> ta.Optional[bytes]:
|
4146
|
+
if not (lst := self._lst):
|
4147
|
+
return None
|
4148
|
+
|
4149
|
+
for i, d in enumerate(lst):
|
4150
|
+
if (p := d.find(delim)) >= 0:
|
4151
|
+
return self._chop(i, p + len(delim))
|
4152
|
+
|
4153
|
+
return None
|
4154
|
+
|
4155
|
+
|
4156
|
+
class IncrementalWriteBuffer:
|
4157
|
+
def __init__(
|
4158
|
+
self,
|
4159
|
+
data: bytes,
|
4160
|
+
*,
|
4161
|
+
write_size: int = 0x10000,
|
4162
|
+
) -> None:
|
4163
|
+
super().__init__()
|
4164
|
+
|
4165
|
+
check_non_empty(data)
|
4166
|
+
self._len = len(data)
|
4167
|
+
self._write_size = write_size
|
4168
|
+
|
4169
|
+
self._lst = [
|
4170
|
+
data[i:i + write_size]
|
4171
|
+
for i in range(0, len(data), write_size)
|
4172
|
+
]
|
4173
|
+
self._pos = 0
|
4174
|
+
|
4175
|
+
@property
|
4176
|
+
def rem(self) -> int:
|
4177
|
+
return self._len - self._pos
|
4178
|
+
|
4179
|
+
def write(self, fn: ta.Callable[[bytes], int]) -> int:
|
4180
|
+
lst = check_non_empty(self._lst)
|
4181
|
+
|
4182
|
+
t = 0
|
4183
|
+
for i, d in enumerate(lst): # noqa
|
4184
|
+
n = fn(check_non_empty(d))
|
4185
|
+
if not n:
|
4186
|
+
break
|
4187
|
+
t += n
|
4188
|
+
|
4189
|
+
if t:
|
4190
|
+
self._lst = [
|
4191
|
+
*([d[n:]] if n < len(d) else []),
|
4192
|
+
*lst[i + 1:],
|
4193
|
+
]
|
4194
|
+
self._pos += t
|
4195
|
+
|
4196
|
+
return t
|
4197
|
+
|
4198
|
+
|
3464
4199
|
########################################
|
3465
4200
|
# ../../../omlish/lite/journald.py
|
3466
4201
|
|
@@ -4601,239 +5336,6 @@ def parse_logging_level(value: ta.Union[str, int]) -> int:
|
|
4601
5336
|
return level
|
4602
5337
|
|
4603
5338
|
|
4604
|
-
########################################
|
4605
|
-
# ../poller.py
|
4606
|
-
|
4607
|
-
|
4608
|
-
class Poller(DaemonizeListener, abc.ABC):
|
4609
|
-
def __init__(self) -> None:
|
4610
|
-
super().__init__()
|
4611
|
-
|
4612
|
-
@abc.abstractmethod
|
4613
|
-
def register_readable(self, fd: Fd) -> None:
|
4614
|
-
raise NotImplementedError
|
4615
|
-
|
4616
|
-
@abc.abstractmethod
|
4617
|
-
def register_writable(self, fd: Fd) -> None:
|
4618
|
-
raise NotImplementedError
|
4619
|
-
|
4620
|
-
@abc.abstractmethod
|
4621
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4622
|
-
raise NotImplementedError
|
4623
|
-
|
4624
|
-
@abc.abstractmethod
|
4625
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4626
|
-
raise NotImplementedError
|
4627
|
-
|
4628
|
-
@abc.abstractmethod
|
4629
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4630
|
-
raise NotImplementedError
|
4631
|
-
|
4632
|
-
def before_daemonize(self) -> None: # noqa
|
4633
|
-
pass
|
4634
|
-
|
4635
|
-
def after_daemonize(self) -> None: # noqa
|
4636
|
-
pass
|
4637
|
-
|
4638
|
-
def close(self) -> None: # noqa
|
4639
|
-
pass
|
4640
|
-
|
4641
|
-
|
4642
|
-
class SelectPoller(Poller):
|
4643
|
-
def __init__(self) -> None:
|
4644
|
-
super().__init__()
|
4645
|
-
|
4646
|
-
self._readable: ta.Set[Fd] = set()
|
4647
|
-
self._writable: ta.Set[Fd] = set()
|
4648
|
-
|
4649
|
-
def register_readable(self, fd: Fd) -> None:
|
4650
|
-
self._readable.add(fd)
|
4651
|
-
|
4652
|
-
def register_writable(self, fd: Fd) -> None:
|
4653
|
-
self._writable.add(fd)
|
4654
|
-
|
4655
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4656
|
-
self._readable.discard(fd)
|
4657
|
-
|
4658
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4659
|
-
self._writable.discard(fd)
|
4660
|
-
|
4661
|
-
def unregister_all(self) -> None:
|
4662
|
-
self._readable.clear()
|
4663
|
-
self._writable.clear()
|
4664
|
-
|
4665
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4666
|
-
try:
|
4667
|
-
r, w, x = select.select(
|
4668
|
-
self._readable,
|
4669
|
-
self._writable,
|
4670
|
-
[], timeout,
|
4671
|
-
)
|
4672
|
-
except OSError as exc:
|
4673
|
-
if exc.args[0] == errno.EINTR:
|
4674
|
-
log.debug('EINTR encountered in poll')
|
4675
|
-
return [], []
|
4676
|
-
if exc.args[0] == errno.EBADF:
|
4677
|
-
log.debug('EBADF encountered in poll')
|
4678
|
-
self.unregister_all()
|
4679
|
-
return [], []
|
4680
|
-
raise
|
4681
|
-
return r, w
|
4682
|
-
|
4683
|
-
|
4684
|
-
class PollPoller(Poller):
|
4685
|
-
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
4686
|
-
_WRITE = select.POLLOUT
|
4687
|
-
|
4688
|
-
def __init__(self) -> None:
|
4689
|
-
super().__init__()
|
4690
|
-
|
4691
|
-
self._poller = select.poll()
|
4692
|
-
self._readable: set[Fd] = set()
|
4693
|
-
self._writable: set[Fd] = set()
|
4694
|
-
|
4695
|
-
def register_readable(self, fd: Fd) -> None:
|
4696
|
-
self._poller.register(fd, self._READ)
|
4697
|
-
self._readable.add(fd)
|
4698
|
-
|
4699
|
-
def register_writable(self, fd: Fd) -> None:
|
4700
|
-
self._poller.register(fd, self._WRITE)
|
4701
|
-
self._writable.add(fd)
|
4702
|
-
|
4703
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4704
|
-
self._readable.discard(fd)
|
4705
|
-
self._poller.unregister(fd)
|
4706
|
-
if fd in self._writable:
|
4707
|
-
self._poller.register(fd, self._WRITE)
|
4708
|
-
|
4709
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4710
|
-
self._writable.discard(fd)
|
4711
|
-
self._poller.unregister(fd)
|
4712
|
-
if fd in self._readable:
|
4713
|
-
self._poller.register(fd, self._READ)
|
4714
|
-
|
4715
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4716
|
-
fds = self._poll_fds(timeout) # type: ignore
|
4717
|
-
readable, writable = [], []
|
4718
|
-
for fd, eventmask in fds:
|
4719
|
-
if self._ignore_invalid(fd, eventmask):
|
4720
|
-
continue
|
4721
|
-
if eventmask & self._READ:
|
4722
|
-
readable.append(fd)
|
4723
|
-
if eventmask & self._WRITE:
|
4724
|
-
writable.append(fd)
|
4725
|
-
return readable, writable
|
4726
|
-
|
4727
|
-
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[Fd, Fd]]:
|
4728
|
-
try:
|
4729
|
-
return self._poller.poll(timeout * 1000) # type: ignore
|
4730
|
-
except OSError as exc:
|
4731
|
-
if exc.args[0] == errno.EINTR:
|
4732
|
-
log.debug('EINTR encountered in poll')
|
4733
|
-
return []
|
4734
|
-
raise
|
4735
|
-
|
4736
|
-
def _ignore_invalid(self, fd: Fd, eventmask: int) -> bool:
|
4737
|
-
if eventmask & select.POLLNVAL:
|
4738
|
-
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
4739
|
-
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
4740
|
-
self._poller.unregister(fd)
|
4741
|
-
self._readable.discard(fd)
|
4742
|
-
self._writable.discard(fd)
|
4743
|
-
return True
|
4744
|
-
return False
|
4745
|
-
|
4746
|
-
|
4747
|
-
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
4748
|
-
class KqueuePoller(Poller):
|
4749
|
-
max_events = 1000
|
4750
|
-
|
4751
|
-
def __init__(self) -> None:
|
4752
|
-
super().__init__()
|
4753
|
-
|
4754
|
-
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
4755
|
-
self._readable: set[Fd] = set()
|
4756
|
-
self._writable: set[Fd] = set()
|
4757
|
-
|
4758
|
-
def register_readable(self, fd: Fd) -> None:
|
4759
|
-
self._readable.add(fd)
|
4760
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
4761
|
-
self._kqueue_control(fd, kevent)
|
4762
|
-
|
4763
|
-
def register_writable(self, fd: Fd) -> None:
|
4764
|
-
self._writable.add(fd)
|
4765
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
4766
|
-
self._kqueue_control(fd, kevent)
|
4767
|
-
|
4768
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4769
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
4770
|
-
self._readable.discard(fd)
|
4771
|
-
self._kqueue_control(fd, kevent)
|
4772
|
-
|
4773
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4774
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
4775
|
-
self._writable.discard(fd)
|
4776
|
-
self._kqueue_control(fd, kevent)
|
4777
|
-
|
4778
|
-
def _kqueue_control(self, fd: Fd, kevent: 'select.kevent') -> None:
|
4779
|
-
try:
|
4780
|
-
self._kqueue.control([kevent], 0) # type: ignore
|
4781
|
-
except OSError as error:
|
4782
|
-
if error.errno == errno.EBADF:
|
4783
|
-
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
4784
|
-
else:
|
4785
|
-
raise
|
4786
|
-
|
4787
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4788
|
-
readable, writable = [], [] # type: ignore
|
4789
|
-
|
4790
|
-
try:
|
4791
|
-
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
4792
|
-
except OSError as error:
|
4793
|
-
if error.errno == errno.EINTR:
|
4794
|
-
log.debug('EINTR encountered in poll')
|
4795
|
-
return readable, writable
|
4796
|
-
raise
|
4797
|
-
|
4798
|
-
for kevent in kevents:
|
4799
|
-
if kevent.filter == select.KQ_FILTER_READ:
|
4800
|
-
readable.append(kevent.ident)
|
4801
|
-
if kevent.filter == select.KQ_FILTER_WRITE:
|
4802
|
-
writable.append(kevent.ident)
|
4803
|
-
|
4804
|
-
return readable, writable
|
4805
|
-
|
4806
|
-
def before_daemonize(self) -> None:
|
4807
|
-
self.close()
|
4808
|
-
|
4809
|
-
def after_daemonize(self) -> None:
|
4810
|
-
self._kqueue = select.kqueue()
|
4811
|
-
for fd in self._readable:
|
4812
|
-
self.register_readable(fd)
|
4813
|
-
for fd in self._writable:
|
4814
|
-
self.register_writable(fd)
|
4815
|
-
|
4816
|
-
def close(self) -> None:
|
4817
|
-
self._kqueue.close() # type: ignore
|
4818
|
-
self._kqueue = None
|
4819
|
-
|
4820
|
-
else:
|
4821
|
-
KqueuePoller = None
|
4822
|
-
|
4823
|
-
|
4824
|
-
def get_poller_impl() -> ta.Type[Poller]:
|
4825
|
-
if (
|
4826
|
-
(sys.platform == 'darwin' or sys.platform.startswith('freebsd')) and
|
4827
|
-
hasattr(select, 'kqueue') and
|
4828
|
-
KqueuePoller is not None
|
4829
|
-
):
|
4830
|
-
return KqueuePoller
|
4831
|
-
elif hasattr(select, 'poll'):
|
4832
|
-
return PollPoller
|
4833
|
-
else:
|
4834
|
-
return SelectPoller
|
4835
|
-
|
4836
|
-
|
4837
5339
|
########################################
|
4838
5340
|
# ../../../omlish/lite/http/coroserver.py
|
4839
5341
|
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
@@ -5442,96 +5944,45 @@ class SupervisorStateManager(abc.ABC):
|
|
5442
5944
|
##
|
5443
5945
|
|
5444
5946
|
|
5445
|
-
class
|
5446
|
-
@property
|
5947
|
+
class HasDispatchers(abc.ABC):
|
5447
5948
|
@abc.abstractmethod
|
5448
|
-
def
|
5949
|
+
def get_dispatchers(self) -> 'Dispatchers':
|
5449
5950
|
raise NotImplementedError
|
5450
5951
|
|
5952
|
+
|
5953
|
+
class ProcessDispatcher(FdIoHandler, abc.ABC):
|
5451
5954
|
@property
|
5452
5955
|
@abc.abstractmethod
|
5453
|
-
def
|
5956
|
+
def channel(self) -> str:
|
5454
5957
|
raise NotImplementedError
|
5455
5958
|
|
5456
5959
|
@property
|
5457
5960
|
@abc.abstractmethod
|
5458
|
-
def
|
5961
|
+
def process(self) -> 'Process':
|
5459
5962
|
raise NotImplementedError
|
5460
5963
|
|
5461
|
-
#
|
5462
5964
|
|
5965
|
+
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
5463
5966
|
@abc.abstractmethod
|
5464
|
-
def
|
5967
|
+
def remove_logs(self) -> None:
|
5465
5968
|
raise NotImplementedError
|
5466
5969
|
|
5467
5970
|
@abc.abstractmethod
|
5468
|
-
def
|
5971
|
+
def reopen_logs(self) -> None:
|
5469
5972
|
raise NotImplementedError
|
5470
5973
|
|
5471
|
-
#
|
5472
5974
|
|
5975
|
+
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
5473
5976
|
@abc.abstractmethod
|
5474
|
-
def
|
5977
|
+
def write(self, chars: ta.Union[bytes, str]) -> None:
|
5475
5978
|
raise NotImplementedError
|
5476
5979
|
|
5477
5980
|
@abc.abstractmethod
|
5478
|
-
def
|
5981
|
+
def flush(self) -> None:
|
5479
5982
|
raise NotImplementedError
|
5480
5983
|
|
5481
|
-
#
|
5482
5984
|
|
5483
|
-
|
5484
|
-
raise TypeError
|
5485
|
-
|
5486
|
-
def handle_write_event(self) -> None:
|
5487
|
-
raise TypeError
|
5488
|
-
|
5489
|
-
#
|
5490
|
-
|
5491
|
-
def handle_connect(self) -> None:
|
5492
|
-
raise TypeError
|
5493
|
-
|
5494
|
-
def handle_close(self) -> None:
|
5495
|
-
raise TypeError
|
5496
|
-
|
5497
|
-
def handle_accepted(self, sock, addr) -> None:
|
5498
|
-
raise TypeError
|
5499
|
-
|
5500
|
-
|
5501
|
-
class HasDispatchers(abc.ABC):
|
5502
|
-
@abc.abstractmethod
|
5503
|
-
def get_dispatchers(self) -> 'Dispatchers':
|
5504
|
-
raise NotImplementedError
|
5505
|
-
|
5506
|
-
|
5507
|
-
class ProcessDispatcher(Dispatcher, abc.ABC):
|
5508
|
-
@property
|
5509
|
-
@abc.abstractmethod
|
5510
|
-
def process(self) -> 'Process':
|
5511
|
-
raise NotImplementedError
|
5512
|
-
|
5513
|
-
|
5514
|
-
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
5515
|
-
@abc.abstractmethod
|
5516
|
-
def remove_logs(self) -> None:
|
5517
|
-
raise NotImplementedError
|
5518
|
-
|
5519
|
-
@abc.abstractmethod
|
5520
|
-
def reopen_logs(self) -> None:
|
5521
|
-
raise NotImplementedError
|
5522
|
-
|
5523
|
-
|
5524
|
-
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
5525
|
-
@abc.abstractmethod
|
5526
|
-
def write(self, chars: ta.Union[bytes, str]) -> None:
|
5527
|
-
raise NotImplementedError
|
5528
|
-
|
5529
|
-
@abc.abstractmethod
|
5530
|
-
def flush(self) -> None:
|
5531
|
-
raise NotImplementedError
|
5532
|
-
|
5533
|
-
|
5534
|
-
##
|
5985
|
+
##
|
5535
5986
|
|
5536
5987
|
|
5537
5988
|
class Process(
|
@@ -5625,13 +6076,139 @@ class ProcessGroup(
|
|
5625
6076
|
raise NotImplementedError
|
5626
6077
|
|
5627
6078
|
|
6079
|
+
########################################
|
6080
|
+
# ../../../omlish/lite/fdio/corohttp.py
|
6081
|
+
|
6082
|
+
|
6083
|
+
class CoroHttpServerConnectionFdIoHandler(SocketFdIoHandler):
|
6084
|
+
def __init__(
|
6085
|
+
self,
|
6086
|
+
addr: SocketAddress,
|
6087
|
+
sock: socket.socket,
|
6088
|
+
handler: HttpHandler,
|
6089
|
+
*,
|
6090
|
+
read_size: int = 0x10000,
|
6091
|
+
write_size: int = 0x10000,
|
6092
|
+
) -> None:
|
6093
|
+
check_state(not sock.getblocking())
|
6094
|
+
|
6095
|
+
super().__init__(addr, sock)
|
6096
|
+
|
6097
|
+
self._handler = handler
|
6098
|
+
self._read_size = read_size
|
6099
|
+
self._write_size = write_size
|
6100
|
+
|
6101
|
+
self._read_buf = ReadableListBuffer()
|
6102
|
+
self._write_buf: IncrementalWriteBuffer | None = None
|
6103
|
+
|
6104
|
+
self._coro_srv = CoroHttpServer(
|
6105
|
+
addr,
|
6106
|
+
handler=self._handler,
|
6107
|
+
)
|
6108
|
+
self._srv_coro: ta.Optional[ta.Generator[CoroHttpServer.Io, ta.Optional[bytes], None]] = self._coro_srv.coro_handle() # noqa
|
6109
|
+
|
6110
|
+
self._cur_io: CoroHttpServer.Io | None = None
|
6111
|
+
self._next_io()
|
6112
|
+
|
6113
|
+
#
|
6114
|
+
|
6115
|
+
def _next_io(self) -> None: # noqa
|
6116
|
+
coro = check_not_none(self._srv_coro)
|
6117
|
+
|
6118
|
+
d: bytes | None = None
|
6119
|
+
o = self._cur_io
|
6120
|
+
while True:
|
6121
|
+
if o is None:
|
6122
|
+
try:
|
6123
|
+
if d is not None:
|
6124
|
+
o = coro.send(d)
|
6125
|
+
d = None
|
6126
|
+
else:
|
6127
|
+
o = next(coro)
|
6128
|
+
except StopIteration:
|
6129
|
+
self.close()
|
6130
|
+
o = None
|
6131
|
+
break
|
6132
|
+
|
6133
|
+
if isinstance(o, CoroHttpServer.AnyLogIo):
|
6134
|
+
print(o)
|
6135
|
+
o = None
|
6136
|
+
|
6137
|
+
elif isinstance(o, CoroHttpServer.ReadIo):
|
6138
|
+
if (d := self._read_buf.read(o.sz)) is None:
|
6139
|
+
break
|
6140
|
+
o = None
|
6141
|
+
|
6142
|
+
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
6143
|
+
if (d := self._read_buf.read_until(b'\n')) is None:
|
6144
|
+
break
|
6145
|
+
o = None
|
6146
|
+
|
6147
|
+
elif isinstance(o, CoroHttpServer.WriteIo):
|
6148
|
+
check_none(self._write_buf)
|
6149
|
+
self._write_buf = IncrementalWriteBuffer(o.data, write_size=self._write_size)
|
6150
|
+
break
|
6151
|
+
|
6152
|
+
else:
|
6153
|
+
raise TypeError(o)
|
6154
|
+
|
6155
|
+
self._cur_io = o
|
6156
|
+
|
6157
|
+
#
|
6158
|
+
|
6159
|
+
def readable(self) -> bool:
|
6160
|
+
return True
|
6161
|
+
|
6162
|
+
def writable(self) -> bool:
|
6163
|
+
return self._write_buf is not None
|
6164
|
+
|
6165
|
+
#
|
6166
|
+
|
6167
|
+
def on_readable(self) -> None:
|
6168
|
+
try:
|
6169
|
+
buf = check_not_none(self._sock).recv(self._read_size)
|
6170
|
+
except BlockingIOError:
|
6171
|
+
return
|
6172
|
+
except ConnectionResetError:
|
6173
|
+
self.close()
|
6174
|
+
return
|
6175
|
+
if not buf:
|
6176
|
+
self.close()
|
6177
|
+
return
|
6178
|
+
|
6179
|
+
self._read_buf.feed(buf)
|
6180
|
+
|
6181
|
+
if isinstance(self._cur_io, CoroHttpServer.AnyReadIo):
|
6182
|
+
self._next_io()
|
6183
|
+
|
6184
|
+
def on_writable(self) -> None:
|
6185
|
+
check_isinstance(self._cur_io, CoroHttpServer.WriteIo)
|
6186
|
+
wb = check_not_none(self._write_buf)
|
6187
|
+
while wb.rem > 0:
|
6188
|
+
def send(d: bytes) -> int:
|
6189
|
+
try:
|
6190
|
+
return check_not_none(self._sock).send(d)
|
6191
|
+
except ConnectionResetError:
|
6192
|
+
self.close()
|
6193
|
+
return 0
|
6194
|
+
except BlockingIOError:
|
6195
|
+
return 0
|
6196
|
+
if not wb.write(send):
|
6197
|
+
break
|
6198
|
+
|
6199
|
+
if wb.rem < 1:
|
6200
|
+
self._write_buf = None
|
6201
|
+
self._cur_io = None
|
6202
|
+
self._next_io()
|
6203
|
+
|
6204
|
+
|
5628
6205
|
########################################
|
5629
6206
|
# ../dispatchers.py
|
5630
6207
|
|
5631
6208
|
|
5632
|
-
class Dispatchers(KeyedCollection[Fd,
|
5633
|
-
def _key(self, v:
|
5634
|
-
return v.fd
|
6209
|
+
class Dispatchers(KeyedCollection[Fd, FdIoHandler]):
|
6210
|
+
def _key(self, v: FdIoHandler) -> Fd:
|
6211
|
+
return Fd(v.fd())
|
5635
6212
|
|
5636
6213
|
#
|
5637
6214
|
|
@@ -5640,9 +6217,9 @@ class Dispatchers(KeyedCollection[Fd, Dispatcher]):
|
|
5640
6217
|
# note that we *must* call readable() for every dispatcher, as it may have side effects for a given
|
5641
6218
|
# dispatcher (eg. call handle_listener_state_change for event listener processes)
|
5642
6219
|
if d.readable():
|
5643
|
-
d.
|
6220
|
+
d.on_readable()
|
5644
6221
|
if d.writable():
|
5645
|
-
d.
|
6222
|
+
d.on_writable()
|
5646
6223
|
|
5647
6224
|
#
|
5648
6225
|
|
@@ -5696,7 +6273,6 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
5696
6273
|
def channel(self) -> str:
|
5697
6274
|
return self._channel
|
5698
6275
|
|
5699
|
-
@property
|
5700
6276
|
def fd(self) -> Fd:
|
5701
6277
|
return self._fd
|
5702
6278
|
|
@@ -5711,7 +6287,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
5711
6287
|
log.debug('fd %s closed, stopped monitoring %s', self._fd, self)
|
5712
6288
|
self._closed = True
|
5713
6289
|
|
5714
|
-
def
|
6290
|
+
def on_error(self, exc: ta.Optional[BaseException] = None) -> None:
|
5715
6291
|
nil, t, v, tbinfo = compact_traceback()
|
5716
6292
|
|
5717
6293
|
log.critical('uncaptured python exception, closing channel %s (%s:%s %s)', repr(self), t, v, tbinfo)
|
@@ -5927,7 +6503,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
5927
6503
|
return False
|
5928
6504
|
return True
|
5929
6505
|
|
5930
|
-
def
|
6506
|
+
def on_readable(self) -> None:
|
5931
6507
|
data = read_fd(self._fd)
|
5932
6508
|
self._output_buffer += data
|
5933
6509
|
self.record_output()
|
@@ -5965,15 +6541,12 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
|
|
5965
6541
|
return True
|
5966
6542
|
return False
|
5967
6543
|
|
5968
|
-
def readable(self) -> bool:
|
5969
|
-
return False
|
5970
|
-
|
5971
6544
|
def flush(self) -> None:
|
5972
6545
|
# other code depends on this raising EPIPE if the pipe is closed
|
5973
6546
|
sent = os.write(self._fd, as_bytes(self._input_buffer))
|
5974
6547
|
self._input_buffer = self._input_buffer[sent:]
|
5975
6548
|
|
5976
|
-
def
|
6549
|
+
def on_writable(self) -> None:
|
5977
6550
|
if self._input_buffer:
|
5978
6551
|
try:
|
5979
6552
|
self.flush()
|
@@ -5985,79 +6558,6 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
|
|
5985
6558
|
raise
|
5986
6559
|
|
5987
6560
|
|
5988
|
-
########################################
|
5989
|
-
# ../groups.py
|
5990
|
-
|
5991
|
-
|
5992
|
-
class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
|
5993
|
-
def __init__(
|
5994
|
-
self,
|
5995
|
-
*,
|
5996
|
-
event_callbacks: EventCallbacks,
|
5997
|
-
) -> None:
|
5998
|
-
super().__init__()
|
5999
|
-
|
6000
|
-
self._event_callbacks = event_callbacks
|
6001
|
-
|
6002
|
-
self._by_name: ta.Dict[str, ProcessGroup] = {}
|
6003
|
-
|
6004
|
-
@property
|
6005
|
-
def _by_key(self) -> ta.Mapping[str, ProcessGroup]:
|
6006
|
-
return self._by_name
|
6007
|
-
|
6008
|
-
#
|
6009
|
-
|
6010
|
-
def all_processes(self) -> ta.Iterator[Process]:
|
6011
|
-
for g in self:
|
6012
|
-
yield from g
|
6013
|
-
|
6014
|
-
#
|
6015
|
-
|
6016
|
-
def add(self, group: ProcessGroup) -> None:
|
6017
|
-
if (name := group.name) in self._by_name:
|
6018
|
-
raise KeyError(f'Process group already exists: {name}')
|
6019
|
-
|
6020
|
-
self._by_name[name] = group
|
6021
|
-
|
6022
|
-
self._event_callbacks.notify(ProcessGroupAddedEvent(name))
|
6023
|
-
|
6024
|
-
def remove(self, name: str) -> None:
|
6025
|
-
group = self._by_name[name]
|
6026
|
-
|
6027
|
-
group.before_remove()
|
6028
|
-
|
6029
|
-
del self._by_name[name]
|
6030
|
-
|
6031
|
-
self._event_callbacks.notify(ProcessGroupRemovedEvent(name))
|
6032
|
-
|
6033
|
-
def clear(self) -> None:
|
6034
|
-
# FIXME: events?
|
6035
|
-
self._by_name.clear()
|
6036
|
-
|
6037
|
-
#
|
6038
|
-
|
6039
|
-
class Diff(ta.NamedTuple):
|
6040
|
-
added: ta.List[ProcessGroupConfig]
|
6041
|
-
changed: ta.List[ProcessGroupConfig]
|
6042
|
-
removed: ta.List[ProcessGroupConfig]
|
6043
|
-
|
6044
|
-
def diff(self, new: ta.Sequence[ProcessGroupConfig]) -> Diff:
|
6045
|
-
cur = [group.config for group in self]
|
6046
|
-
|
6047
|
-
cur_by_name = {cfg.name: cfg for cfg in cur}
|
6048
|
-
new_by_name = {cfg.name: cfg for cfg in new}
|
6049
|
-
|
6050
|
-
added = [cand for cand in new if cand.name not in cur_by_name]
|
6051
|
-
removed = [cand for cand in cur if cand.name not in new_by_name]
|
6052
|
-
changed = [cand for cand in new if cand != cur_by_name.get(cand.name, cand)]
|
6053
|
-
|
6054
|
-
return ProcessGroupManager.Diff(
|
6055
|
-
added,
|
6056
|
-
changed,
|
6057
|
-
removed,
|
6058
|
-
)
|
6059
|
-
|
6060
|
-
|
6061
6561
|
########################################
|
6062
6562
|
# ../groupsimpl.py
|
6063
6563
|
|
@@ -6402,6 +6902,92 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
6402
6902
|
os.umask(self._config.umask)
|
6403
6903
|
|
6404
6904
|
|
6905
|
+
########################################
|
6906
|
+
# ../groups.py
|
6907
|
+
|
6908
|
+
|
6909
|
+
class ProcessGroupManager(
|
6910
|
+
KeyedCollectionAccessors[str, ProcessGroup],
|
6911
|
+
HasDispatchers,
|
6912
|
+
):
|
6913
|
+
def __init__(
|
6914
|
+
self,
|
6915
|
+
*,
|
6916
|
+
event_callbacks: EventCallbacks,
|
6917
|
+
) -> None:
|
6918
|
+
super().__init__()
|
6919
|
+
|
6920
|
+
self._event_callbacks = event_callbacks
|
6921
|
+
|
6922
|
+
self._by_name: ta.Dict[str, ProcessGroup] = {}
|
6923
|
+
|
6924
|
+
@property
|
6925
|
+
def _by_key(self) -> ta.Mapping[str, ProcessGroup]:
|
6926
|
+
return self._by_name
|
6927
|
+
|
6928
|
+
#
|
6929
|
+
|
6930
|
+
def all_processes(self) -> ta.Iterator[Process]:
|
6931
|
+
for g in self:
|
6932
|
+
yield from g
|
6933
|
+
|
6934
|
+
#
|
6935
|
+
|
6936
|
+
def get_dispatchers(self) -> Dispatchers:
|
6937
|
+
return Dispatchers(
|
6938
|
+
d
|
6939
|
+
for g in self
|
6940
|
+
for p in g
|
6941
|
+
for d in p.get_dispatchers()
|
6942
|
+
)
|
6943
|
+
|
6944
|
+
#
|
6945
|
+
|
6946
|
+
def add(self, group: ProcessGroup) -> None:
|
6947
|
+
if (name := group.name) in self._by_name:
|
6948
|
+
raise KeyError(f'Process group already exists: {name}')
|
6949
|
+
|
6950
|
+
self._by_name[name] = group
|
6951
|
+
|
6952
|
+
self._event_callbacks.notify(ProcessGroupAddedEvent(name))
|
6953
|
+
|
6954
|
+
def remove(self, name: str) -> None:
|
6955
|
+
group = self._by_name[name]
|
6956
|
+
|
6957
|
+
group.before_remove()
|
6958
|
+
|
6959
|
+
del self._by_name[name]
|
6960
|
+
|
6961
|
+
self._event_callbacks.notify(ProcessGroupRemovedEvent(name))
|
6962
|
+
|
6963
|
+
def clear(self) -> None:
|
6964
|
+
# FIXME: events?
|
6965
|
+
self._by_name.clear()
|
6966
|
+
|
6967
|
+
#
|
6968
|
+
|
6969
|
+
class Diff(ta.NamedTuple):
|
6970
|
+
added: ta.List[ProcessGroupConfig]
|
6971
|
+
changed: ta.List[ProcessGroupConfig]
|
6972
|
+
removed: ta.List[ProcessGroupConfig]
|
6973
|
+
|
6974
|
+
def diff(self, new: ta.Sequence[ProcessGroupConfig]) -> Diff:
|
6975
|
+
cur = [group.config for group in self]
|
6976
|
+
|
6977
|
+
cur_by_name = {cfg.name: cfg for cfg in cur}
|
6978
|
+
new_by_name = {cfg.name: cfg for cfg in new}
|
6979
|
+
|
6980
|
+
added = [cand for cand in new if cand.name not in cur_by_name]
|
6981
|
+
removed = [cand for cand in cur if cand.name not in new_by_name]
|
6982
|
+
changed = [cand for cand in new if cand != cur_by_name.get(cand.name, cand)]
|
6983
|
+
|
6984
|
+
return ProcessGroupManager.Diff(
|
6985
|
+
added,
|
6986
|
+
changed,
|
6987
|
+
removed,
|
6988
|
+
)
|
6989
|
+
|
6990
|
+
|
6405
6991
|
########################################
|
6406
6992
|
# ../io.py
|
6407
6993
|
|
@@ -6409,49 +6995,59 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
6409
6995
|
##
|
6410
6996
|
|
6411
6997
|
|
6412
|
-
|
6998
|
+
HasDispatchersList = ta.NewType('HasDispatchersList', ta.Sequence[HasDispatchers])
|
6999
|
+
|
7000
|
+
|
7001
|
+
class IoManager(HasDispatchers):
|
6413
7002
|
def __init__(
|
6414
7003
|
self,
|
6415
7004
|
*,
|
6416
|
-
poller:
|
6417
|
-
|
7005
|
+
poller: FdIoPoller,
|
7006
|
+
has_dispatchers_list: HasDispatchersList,
|
6418
7007
|
) -> None:
|
6419
7008
|
super().__init__()
|
6420
7009
|
|
6421
7010
|
self._poller = poller
|
6422
|
-
self.
|
7011
|
+
self._has_dispatchers_list = has_dispatchers_list
|
6423
7012
|
|
6424
7013
|
def get_dispatchers(self) -> Dispatchers:
|
6425
7014
|
return Dispatchers(
|
6426
7015
|
d
|
6427
|
-
for
|
6428
|
-
for d in
|
7016
|
+
for hd in self._has_dispatchers_list
|
7017
|
+
for d in hd.get_dispatchers()
|
6429
7018
|
)
|
6430
7019
|
|
6431
7020
|
def poll(self) -> None:
|
6432
7021
|
dispatchers = self.get_dispatchers()
|
6433
7022
|
|
6434
|
-
|
6435
|
-
if
|
6436
|
-
|
6437
|
-
|
6438
|
-
self._poller.register_writable(fd)
|
7023
|
+
self._poller.update(
|
7024
|
+
{fd for fd, d in dispatchers.items() if d.readable()},
|
7025
|
+
{fd for fd, d in dispatchers.items() if d.writable()},
|
7026
|
+
)
|
6439
7027
|
|
6440
7028
|
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
6441
|
-
|
6442
|
-
|
6443
|
-
|
7029
|
+
log.info(f'Polling: {timeout=}') # noqa
|
7030
|
+
polled = self._poller.poll(timeout)
|
7031
|
+
log.info(f'Polled: {polled=}') # noqa
|
7032
|
+
if polled.msg is not None:
|
7033
|
+
log.error(polled.msg)
|
7034
|
+
if polled.exc is not None:
|
7035
|
+
log.error('Poll exception: %r', polled.exc)
|
7036
|
+
|
7037
|
+
for r in polled.r:
|
7038
|
+
fd = Fd(r)
|
6444
7039
|
if fd in dispatchers:
|
7040
|
+
dispatcher = dispatchers[fd]
|
6445
7041
|
try:
|
6446
|
-
dispatcher = dispatchers[fd]
|
6447
7042
|
log.debug('read event caused by %r', dispatcher)
|
6448
|
-
dispatcher.
|
7043
|
+
dispatcher.on_readable()
|
6449
7044
|
if not dispatcher.readable():
|
6450
7045
|
self._poller.unregister_readable(fd)
|
6451
7046
|
except ExitNow:
|
6452
7047
|
raise
|
6453
|
-
except Exception: # noqa
|
6454
|
-
|
7048
|
+
except Exception as exc: # noqa
|
7049
|
+
log.exception('Error in dispatcher: %r', dispatcher)
|
7050
|
+
dispatcher.on_error(exc)
|
6455
7051
|
else:
|
6456
7052
|
# if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
|
6457
7053
|
# time, which may cause 100% cpu usage
|
@@ -6461,18 +7057,20 @@ class IoManager:
|
|
6461
7057
|
except Exception: # noqa
|
6462
7058
|
pass
|
6463
7059
|
|
6464
|
-
for
|
7060
|
+
for w in polled.w:
|
7061
|
+
fd = Fd(w)
|
6465
7062
|
if fd in dispatchers:
|
7063
|
+
dispatcher = dispatchers[fd]
|
6466
7064
|
try:
|
6467
|
-
dispatcher = dispatchers[fd]
|
6468
7065
|
log.debug('write event caused by %r', dispatcher)
|
6469
|
-
dispatcher.
|
7066
|
+
dispatcher.on_writable()
|
6470
7067
|
if not dispatcher.writable():
|
6471
7068
|
self._poller.unregister_writable(fd)
|
6472
7069
|
except ExitNow:
|
6473
7070
|
raise
|
6474
|
-
except Exception: # noqa
|
6475
|
-
|
7071
|
+
except Exception as exc: # noqa
|
7072
|
+
log.exception('Error in dispatcher: %r', dispatcher)
|
7073
|
+
dispatcher.on_error(exc)
|
6476
7074
|
else:
|
6477
7075
|
log.debug('unexpected write event from fd %r', fd)
|
6478
7076
|
try:
|
@@ -6481,65 +7079,6 @@ class IoManager:
|
|
6481
7079
|
pass
|
6482
7080
|
|
6483
7081
|
|
6484
|
-
########################################
|
6485
|
-
# ../signals.py
|
6486
|
-
|
6487
|
-
|
6488
|
-
class SignalHandler:
|
6489
|
-
def __init__(
|
6490
|
-
self,
|
6491
|
-
*,
|
6492
|
-
states: SupervisorStateManager,
|
6493
|
-
signal_receiver: SignalReceiver,
|
6494
|
-
process_groups: ProcessGroupManager,
|
6495
|
-
) -> None:
|
6496
|
-
super().__init__()
|
6497
|
-
|
6498
|
-
self._states = states
|
6499
|
-
self._signal_receiver = signal_receiver
|
6500
|
-
self._process_groups = process_groups
|
6501
|
-
|
6502
|
-
def set_signals(self) -> None:
|
6503
|
-
self._signal_receiver.install(
|
6504
|
-
signal.SIGTERM,
|
6505
|
-
signal.SIGINT,
|
6506
|
-
signal.SIGQUIT,
|
6507
|
-
signal.SIGHUP,
|
6508
|
-
signal.SIGCHLD,
|
6509
|
-
signal.SIGUSR2,
|
6510
|
-
)
|
6511
|
-
|
6512
|
-
def handle_signals(self) -> None:
|
6513
|
-
sig = self._signal_receiver.get_signal()
|
6514
|
-
if not sig:
|
6515
|
-
return
|
6516
|
-
|
6517
|
-
if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
|
6518
|
-
log.warning('received %s indicating exit request', sig_name(sig))
|
6519
|
-
self._states.set_state(SupervisorState.SHUTDOWN)
|
6520
|
-
|
6521
|
-
elif sig == signal.SIGHUP:
|
6522
|
-
if self._states.state == SupervisorState.SHUTDOWN:
|
6523
|
-
log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig)) # noqa
|
6524
|
-
else:
|
6525
|
-
log.warning('received %s indicating restart request', sig_name(sig)) # noqa
|
6526
|
-
self._states.set_state(SupervisorState.RESTARTING)
|
6527
|
-
|
6528
|
-
elif sig == signal.SIGCHLD:
|
6529
|
-
log.debug('received %s indicating a child quit', sig_name(sig))
|
6530
|
-
|
6531
|
-
elif sig == signal.SIGUSR2:
|
6532
|
-
log.info('received %s indicating log reopen request', sig_name(sig))
|
6533
|
-
|
6534
|
-
for p in self._process_groups.all_processes():
|
6535
|
-
for d in p.get_dispatchers():
|
6536
|
-
if isinstance(d, ProcessOutputDispatcher):
|
6537
|
-
d.reopen_logs()
|
6538
|
-
|
6539
|
-
else:
|
6540
|
-
log.debug('received %s indicating nothing', sig_name(sig))
|
6541
|
-
|
6542
|
-
|
6543
7082
|
########################################
|
6544
7083
|
# ../spawning.py
|
6545
7084
|
|
@@ -6568,6 +7107,123 @@ class ProcessSpawning:
|
|
6568
7107
|
raise NotImplementedError
|
6569
7108
|
|
6570
7109
|
|
7110
|
+
########################################
|
7111
|
+
# ../http.py
|
7112
|
+
|
7113
|
+
|
7114
|
+
##
|
7115
|
+
|
7116
|
+
|
7117
|
+
class SocketServerFdIoHandler(SocketFdIoHandler):
|
7118
|
+
def __init__(
|
7119
|
+
self,
|
7120
|
+
addr: SocketAddress,
|
7121
|
+
on_connect: ta.Callable[[socket.socket, SocketAddress], None],
|
7122
|
+
) -> None:
|
7123
|
+
sock = socket.create_server(addr)
|
7124
|
+
sock.setblocking(False)
|
7125
|
+
|
7126
|
+
super().__init__(addr, sock)
|
7127
|
+
|
7128
|
+
self._on_connect = on_connect
|
7129
|
+
|
7130
|
+
sock.listen(1)
|
7131
|
+
|
7132
|
+
def readable(self) -> bool:
|
7133
|
+
return True
|
7134
|
+
|
7135
|
+
def on_readable(self) -> None:
|
7136
|
+
cli_sock, cli_addr = check_not_none(self._sock).accept()
|
7137
|
+
cli_sock.setblocking(False)
|
7138
|
+
|
7139
|
+
self._on_connect(cli_sock, cli_addr)
|
7140
|
+
|
7141
|
+
|
7142
|
+
##
|
7143
|
+
|
7144
|
+
|
7145
|
+
class HttpServer(HasDispatchers):
|
7146
|
+
class Address(ta.NamedTuple):
|
7147
|
+
a: SocketAddress
|
7148
|
+
|
7149
|
+
class Handler(ta.NamedTuple):
|
7150
|
+
h: HttpHandler
|
7151
|
+
|
7152
|
+
def __init__(
|
7153
|
+
self,
|
7154
|
+
handler: Handler,
|
7155
|
+
addr: Address = Address(('localhost', 8000)),
|
7156
|
+
) -> None:
|
7157
|
+
super().__init__()
|
7158
|
+
|
7159
|
+
self._handler = handler.h
|
7160
|
+
self._addr = addr.a
|
7161
|
+
|
7162
|
+
self._server = SocketServerFdIoHandler(self._addr, self._on_connect)
|
7163
|
+
|
7164
|
+
self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
|
7165
|
+
|
7166
|
+
def get_dispatchers(self) -> Dispatchers:
|
7167
|
+
l = []
|
7168
|
+
for c in self._conns:
|
7169
|
+
if not c.closed:
|
7170
|
+
l.append(c)
|
7171
|
+
self._conns = l
|
7172
|
+
return Dispatchers([
|
7173
|
+
self._server,
|
7174
|
+
*l,
|
7175
|
+
])
|
7176
|
+
|
7177
|
+
def _on_connect(self, sock: socket.socket, addr: SocketAddress) -> None:
|
7178
|
+
conn = CoroHttpServerConnectionFdIoHandler(
|
7179
|
+
addr,
|
7180
|
+
sock,
|
7181
|
+
self._handler,
|
7182
|
+
)
|
7183
|
+
|
7184
|
+
self._conns.append(conn)
|
7185
|
+
|
7186
|
+
|
7187
|
+
##
|
7188
|
+
|
7189
|
+
|
7190
|
+
class SupervisorHttpHandler:
|
7191
|
+
def __init__(
|
7192
|
+
self,
|
7193
|
+
*,
|
7194
|
+
groups: ProcessGroupManager,
|
7195
|
+
) -> None:
|
7196
|
+
super().__init__()
|
7197
|
+
|
7198
|
+
self._groups = groups
|
7199
|
+
|
7200
|
+
def handle(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
|
7201
|
+
dct = {
|
7202
|
+
'method': req.method,
|
7203
|
+
'path': req.path,
|
7204
|
+
'data': len(req.data or b''),
|
7205
|
+
'groups': {
|
7206
|
+
g.name: {
|
7207
|
+
'processes': {
|
7208
|
+
p.name: {
|
7209
|
+
'pid': p.pid,
|
7210
|
+
}
|
7211
|
+
for p in g
|
7212
|
+
},
|
7213
|
+
}
|
7214
|
+
for g in self._groups
|
7215
|
+
},
|
7216
|
+
}
|
7217
|
+
|
7218
|
+
return HttpHandlerResponse(
|
7219
|
+
200,
|
7220
|
+
data=json.dumps(dct, **JSON_PRETTY_KWARGS).encode('utf-8') + b'\n',
|
7221
|
+
headers={
|
7222
|
+
'Content-Type': 'application/json',
|
7223
|
+
},
|
7224
|
+
)
|
7225
|
+
|
7226
|
+
|
6571
7227
|
########################################
|
6572
7228
|
# ../processimpl.py
|
6573
7229
|
|
@@ -6918,6 +7574,7 @@ class ProcessImpl(Process):
|
|
6918
7574
|
self._last_stop = now
|
6919
7575
|
|
6920
7576
|
if now > self._last_start:
|
7577
|
+
log.info(f'{now - self._last_start=}') # noqa
|
6921
7578
|
too_quickly = now - self._last_start < self._config.startsecs
|
6922
7579
|
else:
|
6923
7580
|
too_quickly = False
|
@@ -6987,8 +7644,6 @@ class ProcessImpl(Process):
|
|
6987
7644
|
|
6988
7645
|
self._check_and_adjust_for_system_clock_rollback(now)
|
6989
7646
|
|
6990
|
-
logger = log
|
6991
|
-
|
6992
7647
|
if self._supervisor_states.state > SupervisorState.RESTARTING:
|
6993
7648
|
# dont start any processes if supervisor is shutting down
|
6994
7649
|
if state == ProcessState.EXITED:
|
@@ -7020,14 +7675,14 @@ class ProcessImpl(Process):
|
|
7020
7675
|
self.check_in_state(ProcessState.STARTING)
|
7021
7676
|
self.change_state(ProcessState.RUNNING)
|
7022
7677
|
msg = ('entered RUNNING state, process has stayed up for > than %s seconds (startsecs)' % self._config.startsecs) # noqa
|
7023
|
-
|
7678
|
+
log.info('success: %s %s', self.name, msg)
|
7024
7679
|
|
7025
7680
|
if state == ProcessState.BACKOFF:
|
7026
7681
|
if self._backoff > self._config.startretries:
|
7027
7682
|
# BACKOFF -> FATAL if the proc has exceeded its number of retries
|
7028
7683
|
self.give_up()
|
7029
7684
|
msg = ('entered FATAL state, too many start retries too quickly')
|
7030
|
-
|
7685
|
+
log.info('gave up: %s %s', self.name, msg)
|
7031
7686
|
|
7032
7687
|
elif state == ProcessState.STOPPING:
|
7033
7688
|
time_left = self._delay - now
|
@@ -7049,6 +7704,65 @@ class ProcessImpl(Process):
|
|
7049
7704
|
pass
|
7050
7705
|
|
7051
7706
|
|
7707
|
+
########################################
|
7708
|
+
# ../signals.py
|
7709
|
+
|
7710
|
+
|
7711
|
+
class SignalHandler:
|
7712
|
+
def __init__(
|
7713
|
+
self,
|
7714
|
+
*,
|
7715
|
+
states: SupervisorStateManager,
|
7716
|
+
signal_receiver: SignalReceiver,
|
7717
|
+
process_groups: ProcessGroupManager,
|
7718
|
+
) -> None:
|
7719
|
+
super().__init__()
|
7720
|
+
|
7721
|
+
self._states = states
|
7722
|
+
self._signal_receiver = signal_receiver
|
7723
|
+
self._process_groups = process_groups
|
7724
|
+
|
7725
|
+
def set_signals(self) -> None:
|
7726
|
+
self._signal_receiver.install(
|
7727
|
+
signal.SIGTERM,
|
7728
|
+
signal.SIGINT,
|
7729
|
+
signal.SIGQUIT,
|
7730
|
+
signal.SIGHUP,
|
7731
|
+
signal.SIGCHLD,
|
7732
|
+
signal.SIGUSR2,
|
7733
|
+
)
|
7734
|
+
|
7735
|
+
def handle_signals(self) -> None:
|
7736
|
+
sig = self._signal_receiver.get_signal()
|
7737
|
+
if not sig:
|
7738
|
+
return
|
7739
|
+
|
7740
|
+
if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
|
7741
|
+
log.warning('received %s indicating exit request', sig_name(sig))
|
7742
|
+
self._states.set_state(SupervisorState.SHUTDOWN)
|
7743
|
+
|
7744
|
+
elif sig == signal.SIGHUP:
|
7745
|
+
if self._states.state == SupervisorState.SHUTDOWN:
|
7746
|
+
log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig)) # noqa
|
7747
|
+
else:
|
7748
|
+
log.warning('received %s indicating restart request', sig_name(sig)) # noqa
|
7749
|
+
self._states.set_state(SupervisorState.RESTARTING)
|
7750
|
+
|
7751
|
+
elif sig == signal.SIGCHLD:
|
7752
|
+
log.debug('received %s indicating a child quit', sig_name(sig))
|
7753
|
+
|
7754
|
+
elif sig == signal.SIGUSR2:
|
7755
|
+
log.info('received %s indicating log reopen request', sig_name(sig))
|
7756
|
+
|
7757
|
+
for p in self._process_groups.all_processes():
|
7758
|
+
for d in p.get_dispatchers():
|
7759
|
+
if isinstance(d, ProcessOutputDispatcher):
|
7760
|
+
d.reopen_logs()
|
7761
|
+
|
7762
|
+
else:
|
7763
|
+
log.debug('received %s indicating nothing', sig_name(sig))
|
7764
|
+
|
7765
|
+
|
7052
7766
|
########################################
|
7053
7767
|
# ../spawningimpl.py
|
7054
7768
|
|
@@ -7209,7 +7923,7 @@ class ProcessSpawningImpl(ProcessSpawning):
|
|
7209
7923
|
return exe, args
|
7210
7924
|
|
7211
7925
|
def _make_dispatchers(self, pipes: ProcessPipes) -> Dispatchers:
|
7212
|
-
dispatchers: ta.List[
|
7926
|
+
dispatchers: ta.List[FdIoHandler] = []
|
7213
7927
|
|
7214
7928
|
if pipes.stdout is not None:
|
7215
7929
|
dispatchers.append(check_isinstance(self._output_dispatcher_factory(
|
@@ -7399,7 +8113,7 @@ class Supervisor:
|
|
7399
8113
|
self,
|
7400
8114
|
*,
|
7401
8115
|
config: ServerConfig,
|
7402
|
-
poller:
|
8116
|
+
poller: FdIoPoller,
|
7403
8117
|
process_groups: ProcessGroupManager,
|
7404
8118
|
signal_handler: SignalHandler,
|
7405
8119
|
event_callbacks: EventCallbacks,
|
@@ -7515,7 +8229,9 @@ class Supervisor:
|
|
7515
8229
|
#
|
7516
8230
|
|
7517
8231
|
def _run_once(self) -> None:
|
8232
|
+
now = time.time()
|
7518
8233
|
self._poll()
|
8234
|
+
log.info(f'Poll took {time.time() - now}') # noqa
|
7519
8235
|
self._reap()
|
7520
8236
|
self._signal_handler.handle_signals()
|
7521
8237
|
self._tick()
|
@@ -7567,6 +8283,7 @@ class Supervisor:
|
|
7567
8283
|
return
|
7568
8284
|
|
7569
8285
|
wp = waitpid()
|
8286
|
+
log.info(f'Waited pid: {wp}') # noqa
|
7570
8287
|
if wp is None or not wp.pid:
|
7571
8288
|
return
|
7572
8289
|
|
@@ -7635,6 +8352,17 @@ def waitpid() -> ta.Optional[WaitedPid]:
|
|
7635
8352
|
# ../inject.py
|
7636
8353
|
|
7637
8354
|
|
8355
|
+
@dc.dataclass(frozen=True)
|
8356
|
+
class _FdIoPollerDaemonizeListener(DaemonizeListener):
|
8357
|
+
_poller: FdIoPoller
|
8358
|
+
|
8359
|
+
def before_daemonize(self) -> None:
|
8360
|
+
self._poller.close()
|
8361
|
+
|
8362
|
+
def after_daemonize(self) -> None:
|
8363
|
+
self._poller.reopen()
|
8364
|
+
|
8365
|
+
|
7638
8366
|
def bind_server(
|
7639
8367
|
config: ServerConfig,
|
7640
8368
|
*,
|
@@ -7644,22 +8372,24 @@ def bind_server(
|
|
7644
8372
|
lst: ta.List[InjectorBindingOrBindings] = [
|
7645
8373
|
inj.bind(config),
|
7646
8374
|
|
8375
|
+
inj.bind_array(DaemonizeListener),
|
7647
8376
|
inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
|
7648
8377
|
|
7649
8378
|
inj.bind(SupervisorSetupImpl, singleton=True),
|
7650
8379
|
inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
|
7651
8380
|
|
7652
|
-
inj.bind(DaemonizeListener, array=True, to_key=Poller),
|
7653
|
-
|
7654
8381
|
inj.bind(EventCallbacks, singleton=True),
|
7655
8382
|
|
7656
8383
|
inj.bind(SignalReceiver, singleton=True),
|
7657
8384
|
|
7658
8385
|
inj.bind(IoManager, singleton=True),
|
8386
|
+
inj.bind_array(HasDispatchers),
|
8387
|
+
inj.bind_array_type(HasDispatchers, HasDispatchersList),
|
7659
8388
|
|
7660
8389
|
inj.bind(SignalHandler, singleton=True),
|
7661
8390
|
|
7662
8391
|
inj.bind(ProcessGroupManager, singleton=True),
|
8392
|
+
inj.bind(HasDispatchers, array=True, to_key=ProcessGroupManager),
|
7663
8393
|
|
7664
8394
|
inj.bind(Supervisor, singleton=True),
|
7665
8395
|
|
@@ -7692,7 +8422,26 @@ def bind_server(
|
|
7692
8422
|
|
7693
8423
|
#
|
7694
8424
|
|
7695
|
-
|
8425
|
+
poller_impl = next(filter(None, [
|
8426
|
+
KqueueFdIoPoller,
|
8427
|
+
PollFdIoPoller,
|
8428
|
+
SelectFdIoPoller,
|
8429
|
+
]))
|
8430
|
+
lst.append(inj.bind(poller_impl, key=FdIoPoller, singleton=True))
|
8431
|
+
inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True)
|
8432
|
+
|
8433
|
+
#
|
8434
|
+
|
8435
|
+
def _provide_http_handler(s: SupervisorHttpHandler) -> HttpServer.Handler:
|
8436
|
+
return HttpServer.Handler(s.handle)
|
8437
|
+
|
8438
|
+
lst.extend([
|
8439
|
+
inj.bind(HttpServer, singleton=True, eager=True),
|
8440
|
+
inj.bind(HasDispatchers, array=True, to_key=HttpServer),
|
8441
|
+
|
8442
|
+
inj.bind(SupervisorHttpHandler, singleton=True),
|
8443
|
+
inj.bind(_provide_http_handler),
|
8444
|
+
])
|
7696
8445
|
|
7697
8446
|
#
|
7698
8447
|
|