ominfra 0.0.0.dev129__py3-none-any.whl → 0.0.0.dev130__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- ominfra/deploy/_executor.py +15 -0
- ominfra/pyremote/_runcommands.py +15 -0
- ominfra/scripts/journald2aws.py +112 -0
- ominfra/scripts/supervisor.py +1233 -484
- ominfra/supervisor/dispatchers.py +7 -6
- ominfra/supervisor/dispatchersimpl.py +3 -7
- ominfra/supervisor/groups.py +16 -1
- ominfra/supervisor/http.py +130 -0
- ominfra/supervisor/inject.py +44 -5
- ominfra/supervisor/io.py +39 -24
- ominfra/supervisor/processimpl.py +3 -4
- ominfra/supervisor/spawningimpl.py +2 -2
- ominfra/supervisor/supervisor.py +5 -2
- ominfra/supervisor/types.py +6 -56
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/RECORD +20 -20
- ominfra/supervisor/poller.py +0 -240
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev129.dist-info → ominfra-0.0.0.dev130.dist-info}/top_level.txt +0 -0
ominfra/scripts/supervisor.py
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
# @omlish-lite
|
4
4
|
# @omlish-script
|
5
5
|
# @omlish-amalg-output ../supervisor/main.py
|
6
|
-
# ruff: noqa: N802 UP006 UP007 UP012 UP036
|
6
|
+
# ruff: noqa: N802 U006 UP006 UP007 UP012 UP036
|
7
7
|
# Supervisor is licensed under the following license:
|
8
8
|
#
|
9
9
|
# A copyright notice accompanies this license document that identifies the copyright holders.
|
@@ -102,6 +102,9 @@ V = ta.TypeVar('V')
|
|
102
102
|
# ../../../omlish/lite/cached.py
|
103
103
|
T = ta.TypeVar('T')
|
104
104
|
|
105
|
+
# ../../../omlish/lite/check.py
|
106
|
+
SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
|
107
|
+
|
105
108
|
# ../../../omlish/lite/socket.py
|
106
109
|
SocketAddress = ta.Any
|
107
110
|
SocketHandlerFactory = ta.Callable[[SocketAddress, ta.BinaryIO, ta.BinaryIO], 'SocketHandler']
|
@@ -1523,6 +1526,228 @@ def check_single(vs: ta.Iterable[T]) -> T:
|
|
1523
1526
|
return v
|
1524
1527
|
|
1525
1528
|
|
1529
|
+
def check_empty(v: SizedT) -> SizedT:
|
1530
|
+
if len(v):
|
1531
|
+
raise ValueError(v)
|
1532
|
+
return v
|
1533
|
+
|
1534
|
+
|
1535
|
+
def check_non_empty(v: SizedT) -> SizedT:
|
1536
|
+
if not len(v):
|
1537
|
+
raise ValueError(v)
|
1538
|
+
return v
|
1539
|
+
|
1540
|
+
|
1541
|
+
########################################
|
1542
|
+
# ../../../omlish/lite/fdio/pollers.py
|
1543
|
+
|
1544
|
+
|
1545
|
+
##
|
1546
|
+
|
1547
|
+
|
1548
|
+
class FdIoPoller(abc.ABC):
|
1549
|
+
def __init__(self) -> None:
|
1550
|
+
super().__init__()
|
1551
|
+
|
1552
|
+
self._readable: ta.Set[int] = set()
|
1553
|
+
self._writable: ta.Set[int] = set()
|
1554
|
+
|
1555
|
+
#
|
1556
|
+
|
1557
|
+
def close(self) -> None: # noqa
|
1558
|
+
pass
|
1559
|
+
|
1560
|
+
def reopen(self) -> None: # noqa
|
1561
|
+
pass
|
1562
|
+
|
1563
|
+
#
|
1564
|
+
|
1565
|
+
@property
|
1566
|
+
@ta.final
|
1567
|
+
def readable(self) -> ta.AbstractSet[int]:
|
1568
|
+
return self._readable
|
1569
|
+
|
1570
|
+
@property
|
1571
|
+
@ta.final
|
1572
|
+
def writable(self) -> ta.AbstractSet[int]:
|
1573
|
+
return self._writable
|
1574
|
+
|
1575
|
+
#
|
1576
|
+
|
1577
|
+
@ta.final
|
1578
|
+
def register_readable(self, fd: int) -> bool:
|
1579
|
+
if fd in self._readable:
|
1580
|
+
return False
|
1581
|
+
self._readable.add(fd)
|
1582
|
+
self._register_readable(fd)
|
1583
|
+
return True
|
1584
|
+
|
1585
|
+
@ta.final
|
1586
|
+
def register_writable(self, fd: int) -> bool:
|
1587
|
+
if fd in self._writable:
|
1588
|
+
return False
|
1589
|
+
self._writable.add(fd)
|
1590
|
+
self._register_writable(fd)
|
1591
|
+
return True
|
1592
|
+
|
1593
|
+
@ta.final
|
1594
|
+
def unregister_readable(self, fd: int) -> bool:
|
1595
|
+
if fd not in self._readable:
|
1596
|
+
return False
|
1597
|
+
self._readable.discard(fd)
|
1598
|
+
self._unregister_readable(fd)
|
1599
|
+
return True
|
1600
|
+
|
1601
|
+
@ta.final
|
1602
|
+
def unregister_writable(self, fd: int) -> bool:
|
1603
|
+
if fd not in self._writable:
|
1604
|
+
return False
|
1605
|
+
self._writable.discard(fd)
|
1606
|
+
self._unregister_writable(fd)
|
1607
|
+
return True
|
1608
|
+
|
1609
|
+
#
|
1610
|
+
|
1611
|
+
def _register_readable(self, fd: int) -> None: # noqa
|
1612
|
+
pass
|
1613
|
+
|
1614
|
+
def _register_writable(self, fd: int) -> None: # noqa
|
1615
|
+
pass
|
1616
|
+
|
1617
|
+
def _unregister_readable(self, fd: int) -> None: # noqa
|
1618
|
+
pass
|
1619
|
+
|
1620
|
+
def _unregister_writable(self, fd: int) -> None: # noqa
|
1621
|
+
pass
|
1622
|
+
|
1623
|
+
#
|
1624
|
+
|
1625
|
+
def update(
|
1626
|
+
self,
|
1627
|
+
r: ta.AbstractSet[int],
|
1628
|
+
w: ta.AbstractSet[int],
|
1629
|
+
) -> None:
|
1630
|
+
for f in r - self._readable:
|
1631
|
+
self.register_readable(f)
|
1632
|
+
for f in w - self._writable:
|
1633
|
+
self.register_writable(f)
|
1634
|
+
for f in self._readable - r:
|
1635
|
+
self.unregister_readable(f)
|
1636
|
+
for f in self._writable - w:
|
1637
|
+
self.unregister_writable(f)
|
1638
|
+
|
1639
|
+
#
|
1640
|
+
|
1641
|
+
@dc.dataclass(frozen=True)
|
1642
|
+
class PollResult:
|
1643
|
+
r: ta.Sequence[int] = ()
|
1644
|
+
w: ta.Sequence[int] = ()
|
1645
|
+
|
1646
|
+
inv: ta.Sequence[int] = ()
|
1647
|
+
|
1648
|
+
msg: ta.Optional[str] = None
|
1649
|
+
exc: ta.Optional[BaseException] = None
|
1650
|
+
|
1651
|
+
@abc.abstractmethod
|
1652
|
+
def poll(self, timeout: ta.Optional[float]) -> PollResult:
|
1653
|
+
raise NotImplementedError
|
1654
|
+
|
1655
|
+
|
1656
|
+
##
|
1657
|
+
|
1658
|
+
|
1659
|
+
class SelectFdIoPoller(FdIoPoller):
|
1660
|
+
def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
|
1661
|
+
try:
|
1662
|
+
r, w, x = select.select(
|
1663
|
+
self._readable,
|
1664
|
+
self._writable,
|
1665
|
+
[],
|
1666
|
+
timeout,
|
1667
|
+
)
|
1668
|
+
|
1669
|
+
except OSError as exc:
|
1670
|
+
if exc.errno == errno.EINTR:
|
1671
|
+
return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
|
1672
|
+
elif exc.errno == errno.EBADF:
|
1673
|
+
return FdIoPoller.PollResult(msg='EBADF encountered in poll', exc=exc)
|
1674
|
+
else:
|
1675
|
+
raise
|
1676
|
+
|
1677
|
+
return FdIoPoller.PollResult(r, w)
|
1678
|
+
|
1679
|
+
|
1680
|
+
##
|
1681
|
+
|
1682
|
+
|
1683
|
+
PollFdIoPoller: ta.Optional[ta.Type[FdIoPoller]]
|
1684
|
+
if hasattr(select, 'poll'):
|
1685
|
+
|
1686
|
+
class _PollFdIoPoller(FdIoPoller):
|
1687
|
+
def __init__(self) -> None:
|
1688
|
+
super().__init__()
|
1689
|
+
|
1690
|
+
self._poller = select.poll()
|
1691
|
+
|
1692
|
+
#
|
1693
|
+
|
1694
|
+
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
1695
|
+
_WRITE = select.POLLOUT
|
1696
|
+
|
1697
|
+
def _register_readable(self, fd: int) -> None:
|
1698
|
+
self._update_registration(fd)
|
1699
|
+
|
1700
|
+
def _register_writable(self, fd: int) -> None:
|
1701
|
+
self._update_registration(fd)
|
1702
|
+
|
1703
|
+
def _unregister_readable(self, fd: int) -> None:
|
1704
|
+
self._update_registration(fd)
|
1705
|
+
|
1706
|
+
def _unregister_writable(self, fd: int) -> None:
|
1707
|
+
self._update_registration(fd)
|
1708
|
+
|
1709
|
+
def _update_registration(self, fd: int) -> None:
|
1710
|
+
r = fd in self._readable
|
1711
|
+
w = fd in self._writable
|
1712
|
+
if r or w:
|
1713
|
+
self._poller.register(fd, (self._READ if r else 0) | (self._WRITE if w else 0))
|
1714
|
+
else:
|
1715
|
+
self._poller.unregister(fd)
|
1716
|
+
|
1717
|
+
#
|
1718
|
+
|
1719
|
+
def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
|
1720
|
+
polled: ta.List[ta.Tuple[int, int]]
|
1721
|
+
try:
|
1722
|
+
polled = self._poller.poll(timeout * 1000 if timeout is not None else None)
|
1723
|
+
|
1724
|
+
except OSError as exc:
|
1725
|
+
if exc.errno == errno.EINTR:
|
1726
|
+
return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
|
1727
|
+
else:
|
1728
|
+
raise
|
1729
|
+
|
1730
|
+
r: ta.List[int] = []
|
1731
|
+
w: ta.List[int] = []
|
1732
|
+
inv: ta.List[int] = []
|
1733
|
+
for fd, mask in polled:
|
1734
|
+
if mask & select.POLLNVAL:
|
1735
|
+
self._poller.unregister(fd)
|
1736
|
+
self._readable.discard(fd)
|
1737
|
+
self._writable.discard(fd)
|
1738
|
+
inv.append(fd)
|
1739
|
+
continue
|
1740
|
+
if mask & self._READ:
|
1741
|
+
r.append(fd)
|
1742
|
+
if mask & self._WRITE:
|
1743
|
+
w.append(fd)
|
1744
|
+
return FdIoPoller.PollResult(r, w, inv=inv)
|
1745
|
+
|
1746
|
+
PollFdIoPoller = _PollFdIoPoller
|
1747
|
+
else:
|
1748
|
+
PollFdIoPoller = None
|
1749
|
+
|
1750
|
+
|
1526
1751
|
########################################
|
1527
1752
|
# ../../../omlish/lite/http/versions.py
|
1528
1753
|
|
@@ -1742,6 +1967,73 @@ class SocketHandler(abc.ABC):
|
|
1742
1967
|
raise NotImplementedError
|
1743
1968
|
|
1744
1969
|
|
1970
|
+
########################################
|
1971
|
+
# ../../../omlish/lite/strings.py
|
1972
|
+
|
1973
|
+
|
1974
|
+
##
|
1975
|
+
|
1976
|
+
|
1977
|
+
def camel_case(name: str, lower: bool = False) -> str:
|
1978
|
+
if not name:
|
1979
|
+
return ''
|
1980
|
+
s = ''.join(map(str.capitalize, name.split('_'))) # noqa
|
1981
|
+
if lower:
|
1982
|
+
s = s[0].lower() + s[1:]
|
1983
|
+
return s
|
1984
|
+
|
1985
|
+
|
1986
|
+
def snake_case(name: str) -> str:
|
1987
|
+
uppers: list[int | None] = [i for i, c in enumerate(name) if c.isupper()]
|
1988
|
+
return '_'.join([name[l:r].lower() for l, r in zip([None, *uppers], [*uppers, None])]).strip('_')
|
1989
|
+
|
1990
|
+
|
1991
|
+
##
|
1992
|
+
|
1993
|
+
|
1994
|
+
def is_dunder(name: str) -> bool:
|
1995
|
+
return (
|
1996
|
+
name[:2] == name[-2:] == '__' and
|
1997
|
+
name[2:3] != '_' and
|
1998
|
+
name[-3:-2] != '_' and
|
1999
|
+
len(name) > 4
|
2000
|
+
)
|
2001
|
+
|
2002
|
+
|
2003
|
+
def is_sunder(name: str) -> bool:
|
2004
|
+
return (
|
2005
|
+
name[0] == name[-1] == '_' and
|
2006
|
+
name[1:2] != '_' and
|
2007
|
+
name[-2:-1] != '_' and
|
2008
|
+
len(name) > 2
|
2009
|
+
)
|
2010
|
+
|
2011
|
+
|
2012
|
+
##
|
2013
|
+
|
2014
|
+
|
2015
|
+
def attr_repr(obj: ta.Any, *attrs: str) -> str:
|
2016
|
+
return f'{type(obj).__name__}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
|
2017
|
+
|
2018
|
+
|
2019
|
+
##
|
2020
|
+
|
2021
|
+
|
2022
|
+
FORMAT_NUM_BYTES_SUFFIXES: ta.Sequence[str] = ['B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB']
|
2023
|
+
|
2024
|
+
|
2025
|
+
def format_num_bytes(num_bytes: int) -> str:
|
2026
|
+
for i, suffix in enumerate(FORMAT_NUM_BYTES_SUFFIXES):
|
2027
|
+
value = num_bytes / 1024 ** i
|
2028
|
+
if num_bytes < 1024 ** (i + 1):
|
2029
|
+
if value.is_integer():
|
2030
|
+
return f'{int(value)}{suffix}'
|
2031
|
+
else:
|
2032
|
+
return f'{value:.2f}{suffix}'
|
2033
|
+
|
2034
|
+
return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
|
2035
|
+
|
2036
|
+
|
1745
2037
|
########################################
|
1746
2038
|
# ../../../omlish/lite/typing.py
|
1747
2039
|
|
@@ -2226,6 +2518,169 @@ def get_user(name: str) -> User:
|
|
2226
2518
|
)
|
2227
2519
|
|
2228
2520
|
|
2521
|
+
########################################
|
2522
|
+
# ../../../omlish/lite/fdio/handlers.py
|
2523
|
+
|
2524
|
+
|
2525
|
+
class FdIoHandler(abc.ABC):
|
2526
|
+
@abc.abstractmethod
|
2527
|
+
def fd(self) -> int:
|
2528
|
+
raise NotImplementedError
|
2529
|
+
|
2530
|
+
#
|
2531
|
+
|
2532
|
+
@property
|
2533
|
+
@abc.abstractmethod
|
2534
|
+
def closed(self) -> bool:
|
2535
|
+
raise NotImplementedError
|
2536
|
+
|
2537
|
+
@abc.abstractmethod
|
2538
|
+
def close(self) -> None:
|
2539
|
+
raise NotImplementedError
|
2540
|
+
|
2541
|
+
#
|
2542
|
+
|
2543
|
+
def readable(self) -> bool:
|
2544
|
+
return False
|
2545
|
+
|
2546
|
+
def writable(self) -> bool:
|
2547
|
+
return False
|
2548
|
+
|
2549
|
+
#
|
2550
|
+
|
2551
|
+
def on_readable(self) -> None:
|
2552
|
+
raise TypeError
|
2553
|
+
|
2554
|
+
def on_writable(self) -> None:
|
2555
|
+
raise TypeError
|
2556
|
+
|
2557
|
+
def on_error(self, exc: ta.Optional[BaseException] = None) -> None: # noqa
|
2558
|
+
pass
|
2559
|
+
|
2560
|
+
|
2561
|
+
class SocketFdIoHandler(FdIoHandler, abc.ABC):
|
2562
|
+
def __init__(
|
2563
|
+
self,
|
2564
|
+
addr: SocketAddress,
|
2565
|
+
sock: socket.socket,
|
2566
|
+
) -> None:
|
2567
|
+
super().__init__()
|
2568
|
+
|
2569
|
+
self._addr = addr
|
2570
|
+
self._sock: ta.Optional[socket.socket] = sock
|
2571
|
+
|
2572
|
+
def fd(self) -> int:
|
2573
|
+
return check_not_none(self._sock).fileno()
|
2574
|
+
|
2575
|
+
@property
|
2576
|
+
def closed(self) -> bool:
|
2577
|
+
return self._sock is None
|
2578
|
+
|
2579
|
+
def close(self) -> None:
|
2580
|
+
if self._sock is not None:
|
2581
|
+
self._sock.close()
|
2582
|
+
self._sock = None
|
2583
|
+
|
2584
|
+
|
2585
|
+
########################################
|
2586
|
+
# ../../../omlish/lite/fdio/kqueue.py
|
2587
|
+
|
2588
|
+
|
2589
|
+
KqueueFdIoPoller: ta.Optional[ta.Type[FdIoPoller]]
|
2590
|
+
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
2591
|
+
|
2592
|
+
class _KqueueFdIoPoller(FdIoPoller):
|
2593
|
+
DEFAULT_MAX_EVENTS = 1000
|
2594
|
+
|
2595
|
+
def __init__(
|
2596
|
+
self,
|
2597
|
+
*,
|
2598
|
+
max_events: int = DEFAULT_MAX_EVENTS,
|
2599
|
+
) -> None:
|
2600
|
+
super().__init__()
|
2601
|
+
|
2602
|
+
self._max_events = max_events
|
2603
|
+
|
2604
|
+
self._kqueue: ta.Optional[ta.Any] = None
|
2605
|
+
|
2606
|
+
#
|
2607
|
+
|
2608
|
+
def _get_kqueue(self) -> 'select.kqueue':
|
2609
|
+
if (kq := self._kqueue) is not None:
|
2610
|
+
return kq
|
2611
|
+
kq = select.kqueue()
|
2612
|
+
self._kqueue = kq
|
2613
|
+
return kq
|
2614
|
+
|
2615
|
+
def close(self) -> None:
|
2616
|
+
if self._kqueue is not None:
|
2617
|
+
self._kqueue.close()
|
2618
|
+
self._kqueue = None
|
2619
|
+
|
2620
|
+
def reopen(self) -> None:
|
2621
|
+
for fd in self._readable:
|
2622
|
+
self._register_readable(fd)
|
2623
|
+
for fd in self._writable:
|
2624
|
+
self._register_writable(fd)
|
2625
|
+
|
2626
|
+
#
|
2627
|
+
|
2628
|
+
def _register_readable(self, fd: int) -> None:
|
2629
|
+
self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_ADD)
|
2630
|
+
|
2631
|
+
def _register_writable(self, fd: int) -> None:
|
2632
|
+
self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)
|
2633
|
+
|
2634
|
+
def _unregister_readable(self, fd: int) -> None:
|
2635
|
+
self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE)
|
2636
|
+
|
2637
|
+
def _unregister_writable(self, fd: int) -> None:
|
2638
|
+
self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)
|
2639
|
+
|
2640
|
+
def _control(self, fd: int, filter: int, flags: int) -> None: # noqa
|
2641
|
+
ke = select.kevent(fd, filter=filter, flags=flags)
|
2642
|
+
kq = self._get_kqueue()
|
2643
|
+
try:
|
2644
|
+
kq.control([ke], 0)
|
2645
|
+
|
2646
|
+
except OSError as exc:
|
2647
|
+
if exc.errno == errno.EBADF:
|
2648
|
+
# log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', ke.ident)
|
2649
|
+
pass
|
2650
|
+
elif exc.errno == errno.ENOENT:
|
2651
|
+
# Can happen when trying to remove an already closed socket
|
2652
|
+
pass
|
2653
|
+
else:
|
2654
|
+
raise
|
2655
|
+
|
2656
|
+
#
|
2657
|
+
|
2658
|
+
def poll(self, timeout: ta.Optional[float]) -> FdIoPoller.PollResult:
|
2659
|
+
kq = self._get_kqueue()
|
2660
|
+
try:
|
2661
|
+
kes = kq.control(None, self._max_events, timeout)
|
2662
|
+
|
2663
|
+
except OSError as exc:
|
2664
|
+
if exc.errno == errno.EINTR:
|
2665
|
+
return FdIoPoller.PollResult(msg='EINTR encountered in poll', exc=exc)
|
2666
|
+
else:
|
2667
|
+
raise
|
2668
|
+
|
2669
|
+
r: ta.List[int] = []
|
2670
|
+
w: ta.List[int] = []
|
2671
|
+
for ke in kes:
|
2672
|
+
if ke.filter == select.KQ_FILTER_READ:
|
2673
|
+
r.append(ke.ident)
|
2674
|
+
if ke.filter == select.KQ_FILTER_WRITE:
|
2675
|
+
w.append(ke.ident)
|
2676
|
+
|
2677
|
+
return FdIoPoller.PollResult(r, w)
|
2678
|
+
|
2679
|
+
KqueueFdIoPoller = _KqueueFdIoPoller
|
2680
|
+
else:
|
2681
|
+
KqueueFdIoPoller = None
|
2682
|
+
|
2683
|
+
|
2229
2684
|
########################################
|
2230
2685
|
# ../../../omlish/lite/http/parsing.py
|
2231
2686
|
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
@@ -2628,11 +3083,6 @@ class HttpRequestParser:
|
|
2628
3083
|
|
2629
3084
|
########################################
|
2630
3085
|
# ../../../omlish/lite/inject.py
|
2631
|
-
"""
|
2632
|
-
TODO:
|
2633
|
-
- recursion detection
|
2634
|
-
- bind empty array
|
2635
|
-
"""
|
2636
3086
|
|
2637
3087
|
|
2638
3088
|
###
|
@@ -2925,7 +3375,11 @@ def build_injector_provider_map(bs: InjectorBindings) -> ta.Mapping[InjectorKey,
|
|
2925
3375
|
|
2926
3376
|
for b in bs.bindings():
|
2927
3377
|
if b.key.array:
|
2928
|
-
am.setdefault(b.key, [])
|
3378
|
+
al = am.setdefault(b.key, [])
|
3379
|
+
if isinstance(b.provider, ArrayInjectorProvider):
|
3380
|
+
al.extend(b.provider.ps)
|
3381
|
+
else:
|
3382
|
+
al.append(b.provider)
|
2929
3383
|
else:
|
2930
3384
|
if b.key in pm:
|
2931
3385
|
raise KeyError(b.key)
|
@@ -3073,8 +3527,16 @@ def build_injection_kwargs_target(
|
|
3073
3527
|
_INJECTOR_INJECTOR_KEY: InjectorKey[Injector] = InjectorKey(Injector)
|
3074
3528
|
|
3075
3529
|
|
3076
|
-
|
3077
|
-
|
3530
|
+
@dc.dataclass(frozen=True)
|
3531
|
+
class _InjectorEager:
|
3532
|
+
key: InjectorKey
|
3533
|
+
|
3534
|
+
|
3535
|
+
_INJECTOR_EAGER_ARRAY_KEY: InjectorKey[_InjectorEager] = InjectorKey(_InjectorEager, array=True)
|
3536
|
+
|
3537
|
+
|
3538
|
+
class _Injector(Injector):
|
3539
|
+
def __init__(self, bs: InjectorBindings, p: ta.Optional[Injector] = None) -> None:
|
3078
3540
|
super().__init__()
|
3079
3541
|
|
3080
3542
|
self._bs = check_isinstance(bs, InjectorBindings)
|
@@ -3087,6 +3549,10 @@ class _Injector(Injector):
|
|
3087
3549
|
|
3088
3550
|
self.__cur_req: ta.Optional[_Injector._Request] = None
|
3089
3551
|
|
3552
|
+
if _INJECTOR_EAGER_ARRAY_KEY in self._pfm:
|
3553
|
+
for e in self.provide(_INJECTOR_EAGER_ARRAY_KEY):
|
3554
|
+
self.provide(e.key)
|
3555
|
+
|
3090
3556
|
class _Request:
|
3091
3557
|
def __init__(self, injector: '_Injector') -> None:
|
3092
3558
|
super().__init__()
|
@@ -3244,6 +3710,8 @@ class InjectorBinder:
|
|
3244
3710
|
to_key: ta.Any = None,
|
3245
3711
|
|
3246
3712
|
singleton: bool = False,
|
3713
|
+
|
3714
|
+
eager: bool = False,
|
3247
3715
|
) -> InjectorBindingOrBindings:
|
3248
3716
|
if obj is None or obj is inspect.Parameter.empty:
|
3249
3717
|
raise TypeError(obj)
|
@@ -3317,13 +3785,21 @@ class InjectorBinder:
|
|
3317
3785
|
if singleton:
|
3318
3786
|
provider = SingletonInjectorProvider(provider)
|
3319
3787
|
|
3788
|
+
binding = InjectorBinding(key, provider)
|
3789
|
+
|
3320
3790
|
##
|
3321
3791
|
|
3322
|
-
|
3792
|
+
extras: ta.List[InjectorBinding] = []
|
3793
|
+
|
3794
|
+
if eager:
|
3795
|
+
extras.append(bind_injector_eager_key(key))
|
3323
3796
|
|
3324
3797
|
##
|
3325
3798
|
|
3326
|
-
|
3799
|
+
if extras:
|
3800
|
+
return as_injector_bindings(binding, *extras)
|
3801
|
+
else:
|
3802
|
+
return binding
|
3327
3803
|
|
3328
3804
|
|
3329
3805
|
###
|
@@ -3346,6 +3822,26 @@ def make_injector_factory(
|
|
3346
3822
|
return outer
|
3347
3823
|
|
3348
3824
|
|
3825
|
+
def bind_injector_array(
|
3826
|
+
obj: ta.Any = None,
|
3827
|
+
*,
|
3828
|
+
tag: ta.Any = None,
|
3829
|
+
) -> InjectorBindingOrBindings:
|
3830
|
+
key = as_injector_key(obj)
|
3831
|
+
if tag is not None:
|
3832
|
+
if key.tag is not None:
|
3833
|
+
raise ValueError('Must not specify multiple tags')
|
3834
|
+
key = dc.replace(key, tag=tag)
|
3835
|
+
|
3836
|
+
if key.array:
|
3837
|
+
raise ValueError('Key must not be array')
|
3838
|
+
|
3839
|
+
return InjectorBinding(
|
3840
|
+
dc.replace(key, array=True),
|
3841
|
+
ArrayInjectorProvider([]),
|
3842
|
+
)
|
3843
|
+
|
3844
|
+
|
3349
3845
|
def make_injector_array_type(
|
3350
3846
|
ele: ta.Union[InjectorKey, InjectorKeyCls],
|
3351
3847
|
cls: U,
|
@@ -3367,6 +3863,10 @@ def make_injector_array_type(
|
|
3367
3863
|
return inner
|
3368
3864
|
|
3369
3865
|
|
3866
|
+
def bind_injector_eager_key(key: ta.Any) -> InjectorBinding:
|
3867
|
+
return InjectorBinding(_INJECTOR_EAGER_ARRAY_KEY, ConstInjectorProvider(_InjectorEager(as_injector_key(key))))
|
3868
|
+
|
3869
|
+
|
3370
3870
|
##
|
3371
3871
|
|
3372
3872
|
|
@@ -3421,6 +3921,8 @@ class Injection:
|
|
3421
3921
|
to_key: ta.Any = None,
|
3422
3922
|
|
3423
3923
|
singleton: bool = False,
|
3924
|
+
|
3925
|
+
eager: bool = False,
|
3424
3926
|
) -> InjectorBindingOrBindings:
|
3425
3927
|
return InjectorBinder.bind(
|
3426
3928
|
obj,
|
@@ -3435,6 +3937,8 @@ class Injection:
|
|
3435
3937
|
to_key=to_key,
|
3436
3938
|
|
3437
3939
|
singleton=singleton,
|
3940
|
+
|
3941
|
+
eager=eager,
|
3438
3942
|
)
|
3439
3943
|
|
3440
3944
|
# helpers
|
@@ -3448,6 +3952,15 @@ class Injection:
|
|
3448
3952
|
) -> InjectorBindingOrBindings:
|
3449
3953
|
return cls.bind(make_injector_factory(fn, cls_, ann))
|
3450
3954
|
|
3955
|
+
@classmethod
|
3956
|
+
def bind_array(
|
3957
|
+
cls,
|
3958
|
+
obj: ta.Any = None,
|
3959
|
+
*,
|
3960
|
+
tag: ta.Any = None,
|
3961
|
+
) -> InjectorBindingOrBindings:
|
3962
|
+
return bind_injector_array(obj, tag=tag)
|
3963
|
+
|
3451
3964
|
@classmethod
|
3452
3965
|
def bind_array_type(
|
3453
3966
|
cls,
|
@@ -3461,6 +3974,228 @@ class Injection:
|
|
3461
3974
|
inj = Injection
|
3462
3975
|
|
3463
3976
|
|
3977
|
+
########################################
|
3978
|
+
# ../../../omlish/lite/io.py
|
3979
|
+
|
3980
|
+
|
3981
|
+
class DelimitingBuffer:
|
3982
|
+
"""
|
3983
|
+
https://github.com/python-trio/trio/issues/796 :|
|
3984
|
+
"""
|
3985
|
+
|
3986
|
+
#
|
3987
|
+
|
3988
|
+
class Error(Exception):
|
3989
|
+
def __init__(self, buffer: 'DelimitingBuffer') -> None:
|
3990
|
+
super().__init__(buffer)
|
3991
|
+
self.buffer = buffer
|
3992
|
+
|
3993
|
+
def __repr__(self) -> str:
|
3994
|
+
return attr_repr(self, 'buffer')
|
3995
|
+
|
3996
|
+
class ClosedError(Error):
|
3997
|
+
pass
|
3998
|
+
|
3999
|
+
#
|
4000
|
+
|
4001
|
+
DEFAULT_DELIMITERS: bytes = b'\n'
|
4002
|
+
|
4003
|
+
def __init__(
|
4004
|
+
self,
|
4005
|
+
delimiters: ta.Iterable[int] = DEFAULT_DELIMITERS,
|
4006
|
+
*,
|
4007
|
+
keep_ends: bool = False,
|
4008
|
+
max_size: ta.Optional[int] = None,
|
4009
|
+
) -> None:
|
4010
|
+
super().__init__()
|
4011
|
+
|
4012
|
+
self._delimiters = frozenset(check_isinstance(d, int) for d in delimiters)
|
4013
|
+
self._keep_ends = keep_ends
|
4014
|
+
self._max_size = max_size
|
4015
|
+
|
4016
|
+
self._buf: ta.Optional[io.BytesIO] = io.BytesIO()
|
4017
|
+
|
4018
|
+
#
|
4019
|
+
|
4020
|
+
@property
|
4021
|
+
def is_closed(self) -> bool:
|
4022
|
+
return self._buf is None
|
4023
|
+
|
4024
|
+
def tell(self) -> int:
|
4025
|
+
if (buf := self._buf) is None:
|
4026
|
+
raise self.ClosedError(self)
|
4027
|
+
return buf.tell()
|
4028
|
+
|
4029
|
+
def peek(self) -> bytes:
|
4030
|
+
if (buf := self._buf) is None:
|
4031
|
+
raise self.ClosedError(self)
|
4032
|
+
return buf.getvalue()
|
4033
|
+
|
4034
|
+
def _find_delim(self, data: ta.Union[bytes, bytearray], i: int) -> ta.Optional[int]:
|
4035
|
+
r = None # type: int | None
|
4036
|
+
for d in self._delimiters:
|
4037
|
+
if (p := data.find(d, i)) >= 0:
|
4038
|
+
if r is None or p < r:
|
4039
|
+
r = p
|
4040
|
+
return r
|
4041
|
+
|
4042
|
+
def _append_and_reset(self, chunk: bytes) -> bytes:
|
4043
|
+
buf = check_not_none(self._buf)
|
4044
|
+
if not buf.tell():
|
4045
|
+
return chunk
|
4046
|
+
|
4047
|
+
buf.write(chunk)
|
4048
|
+
ret = buf.getvalue()
|
4049
|
+
buf.seek(0)
|
4050
|
+
buf.truncate()
|
4051
|
+
return ret
|
4052
|
+
|
4053
|
+
class Incomplete(ta.NamedTuple):
|
4054
|
+
b: bytes
|
4055
|
+
|
4056
|
+
def feed(self, data: ta.Union[bytes, bytearray]) -> ta.Generator[ta.Union[bytes, Incomplete], None, None]:
|
4057
|
+
if (buf := self._buf) is None:
|
4058
|
+
raise self.ClosedError(self)
|
4059
|
+
|
4060
|
+
if not data:
|
4061
|
+
self._buf = None
|
4062
|
+
|
4063
|
+
if buf.tell():
|
4064
|
+
yield self.Incomplete(buf.getvalue())
|
4065
|
+
|
4066
|
+
return
|
4067
|
+
|
4068
|
+
l = len(data)
|
4069
|
+
i = 0
|
4070
|
+
while i < l:
|
4071
|
+
if (p := self._find_delim(data, i)) is None:
|
4072
|
+
break
|
4073
|
+
|
4074
|
+
n = p + 1
|
4075
|
+
if self._keep_ends:
|
4076
|
+
p = n
|
4077
|
+
|
4078
|
+
yield self._append_and_reset(data[i:p])
|
4079
|
+
|
4080
|
+
i = n
|
4081
|
+
|
4082
|
+
if i >= l:
|
4083
|
+
return
|
4084
|
+
|
4085
|
+
if self._max_size is None:
|
4086
|
+
buf.write(data[i:])
|
4087
|
+
return
|
4088
|
+
|
4089
|
+
while i < l:
|
4090
|
+
remaining_data_len = l - i
|
4091
|
+
remaining_buf_capacity = self._max_size - buf.tell()
|
4092
|
+
|
4093
|
+
if remaining_data_len < remaining_buf_capacity:
|
4094
|
+
buf.write(data[i:])
|
4095
|
+
return
|
4096
|
+
|
4097
|
+
p = i + remaining_buf_capacity
|
4098
|
+
yield self.Incomplete(self._append_and_reset(data[i:p]))
|
4099
|
+
i = p
|
4100
|
+
|
4101
|
+
|
4102
|
+
class ReadableListBuffer:
|
4103
|
+
def __init__(self) -> None:
|
4104
|
+
super().__init__()
|
4105
|
+
self._lst: list[bytes] = []
|
4106
|
+
|
4107
|
+
def feed(self, d: bytes) -> None:
|
4108
|
+
if d:
|
4109
|
+
self._lst.append(d)
|
4110
|
+
|
4111
|
+
def _chop(self, i: int, e: int) -> bytes:
|
4112
|
+
lst = self._lst
|
4113
|
+
d = lst[i]
|
4114
|
+
|
4115
|
+
o = b''.join([
|
4116
|
+
*lst[:i],
|
4117
|
+
d[:e],
|
4118
|
+
])
|
4119
|
+
|
4120
|
+
self._lst = [
|
4121
|
+
*([d[e:]] if e < len(d) else []),
|
4122
|
+
*lst[i + 1:],
|
4123
|
+
]
|
4124
|
+
|
4125
|
+
return o
|
4126
|
+
|
4127
|
+
def read(self, n: ta.Optional[int] = None) -> ta.Optional[bytes]:
|
4128
|
+
if n is None:
|
4129
|
+
o = b''.join(self._lst)
|
4130
|
+
self._lst = []
|
4131
|
+
return o
|
4132
|
+
|
4133
|
+
if not (lst := self._lst):
|
4134
|
+
return None
|
4135
|
+
|
4136
|
+
c = 0
|
4137
|
+
for i, d in enumerate(lst):
|
4138
|
+
r = n - c
|
4139
|
+
if (l := len(d)) >= r:
|
4140
|
+
return self._chop(i, r)
|
4141
|
+
c += l
|
4142
|
+
|
4143
|
+
return None
|
4144
|
+
|
4145
|
+
def read_until(self, delim: bytes = b'\n') -> ta.Optional[bytes]:
|
4146
|
+
if not (lst := self._lst):
|
4147
|
+
return None
|
4148
|
+
|
4149
|
+
for i, d in enumerate(lst):
|
4150
|
+
if (p := d.find(delim)) >= 0:
|
4151
|
+
return self._chop(i, p + len(delim))
|
4152
|
+
|
4153
|
+
return None
|
4154
|
+
|
4155
|
+
|
4156
|
+
class IncrementalWriteBuffer:
|
4157
|
+
def __init__(
|
4158
|
+
self,
|
4159
|
+
data: bytes,
|
4160
|
+
*,
|
4161
|
+
write_size: int = 0x10000,
|
4162
|
+
) -> None:
|
4163
|
+
super().__init__()
|
4164
|
+
|
4165
|
+
check_non_empty(data)
|
4166
|
+
self._len = len(data)
|
4167
|
+
self._write_size = write_size
|
4168
|
+
|
4169
|
+
self._lst = [
|
4170
|
+
data[i:i + write_size]
|
4171
|
+
for i in range(0, len(data), write_size)
|
4172
|
+
]
|
4173
|
+
self._pos = 0
|
4174
|
+
|
4175
|
+
@property
|
4176
|
+
def rem(self) -> int:
|
4177
|
+
return self._len - self._pos
|
4178
|
+
|
4179
|
+
def write(self, fn: ta.Callable[[bytes], int]) -> int:
|
4180
|
+
lst = check_non_empty(self._lst)
|
4181
|
+
|
4182
|
+
t = 0
|
4183
|
+
for i, d in enumerate(lst): # noqa
|
4184
|
+
n = fn(check_non_empty(d))
|
4185
|
+
if not n:
|
4186
|
+
break
|
4187
|
+
t += n
|
4188
|
+
|
4189
|
+
if t:
|
4190
|
+
self._lst = [
|
4191
|
+
*([d[n:]] if n < len(d) else []),
|
4192
|
+
*lst[i + 1:],
|
4193
|
+
]
|
4194
|
+
self._pos += t
|
4195
|
+
|
4196
|
+
return t
|
4197
|
+
|
4198
|
+
|
3464
4199
|
########################################
|
3465
4200
|
# ../../../omlish/lite/journald.py
|
3466
4201
|
|
@@ -4601,239 +5336,6 @@ def parse_logging_level(value: ta.Union[str, int]) -> int:
|
|
4601
5336
|
return level
|
4602
5337
|
|
4603
5338
|
|
4604
|
-
########################################
|
4605
|
-
# ../poller.py
|
4606
|
-
|
4607
|
-
|
4608
|
-
class Poller(DaemonizeListener, abc.ABC):
|
4609
|
-
def __init__(self) -> None:
|
4610
|
-
super().__init__()
|
4611
|
-
|
4612
|
-
@abc.abstractmethod
|
4613
|
-
def register_readable(self, fd: Fd) -> None:
|
4614
|
-
raise NotImplementedError
|
4615
|
-
|
4616
|
-
@abc.abstractmethod
|
4617
|
-
def register_writable(self, fd: Fd) -> None:
|
4618
|
-
raise NotImplementedError
|
4619
|
-
|
4620
|
-
@abc.abstractmethod
|
4621
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4622
|
-
raise NotImplementedError
|
4623
|
-
|
4624
|
-
@abc.abstractmethod
|
4625
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4626
|
-
raise NotImplementedError
|
4627
|
-
|
4628
|
-
@abc.abstractmethod
|
4629
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4630
|
-
raise NotImplementedError
|
4631
|
-
|
4632
|
-
def before_daemonize(self) -> None: # noqa
|
4633
|
-
pass
|
4634
|
-
|
4635
|
-
def after_daemonize(self) -> None: # noqa
|
4636
|
-
pass
|
4637
|
-
|
4638
|
-
def close(self) -> None: # noqa
|
4639
|
-
pass
|
4640
|
-
|
4641
|
-
|
4642
|
-
class SelectPoller(Poller):
|
4643
|
-
def __init__(self) -> None:
|
4644
|
-
super().__init__()
|
4645
|
-
|
4646
|
-
self._readable: ta.Set[Fd] = set()
|
4647
|
-
self._writable: ta.Set[Fd] = set()
|
4648
|
-
|
4649
|
-
def register_readable(self, fd: Fd) -> None:
|
4650
|
-
self._readable.add(fd)
|
4651
|
-
|
4652
|
-
def register_writable(self, fd: Fd) -> None:
|
4653
|
-
self._writable.add(fd)
|
4654
|
-
|
4655
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4656
|
-
self._readable.discard(fd)
|
4657
|
-
|
4658
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4659
|
-
self._writable.discard(fd)
|
4660
|
-
|
4661
|
-
def unregister_all(self) -> None:
|
4662
|
-
self._readable.clear()
|
4663
|
-
self._writable.clear()
|
4664
|
-
|
4665
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4666
|
-
try:
|
4667
|
-
r, w, x = select.select(
|
4668
|
-
self._readable,
|
4669
|
-
self._writable,
|
4670
|
-
[], timeout,
|
4671
|
-
)
|
4672
|
-
except OSError as exc:
|
4673
|
-
if exc.args[0] == errno.EINTR:
|
4674
|
-
log.debug('EINTR encountered in poll')
|
4675
|
-
return [], []
|
4676
|
-
if exc.args[0] == errno.EBADF:
|
4677
|
-
log.debug('EBADF encountered in poll')
|
4678
|
-
self.unregister_all()
|
4679
|
-
return [], []
|
4680
|
-
raise
|
4681
|
-
return r, w
|
4682
|
-
|
4683
|
-
|
4684
|
-
class PollPoller(Poller):
|
4685
|
-
_READ = select.POLLIN | select.POLLPRI | select.POLLHUP
|
4686
|
-
_WRITE = select.POLLOUT
|
4687
|
-
|
4688
|
-
def __init__(self) -> None:
|
4689
|
-
super().__init__()
|
4690
|
-
|
4691
|
-
self._poller = select.poll()
|
4692
|
-
self._readable: set[Fd] = set()
|
4693
|
-
self._writable: set[Fd] = set()
|
4694
|
-
|
4695
|
-
def register_readable(self, fd: Fd) -> None:
|
4696
|
-
self._poller.register(fd, self._READ)
|
4697
|
-
self._readable.add(fd)
|
4698
|
-
|
4699
|
-
def register_writable(self, fd: Fd) -> None:
|
4700
|
-
self._poller.register(fd, self._WRITE)
|
4701
|
-
self._writable.add(fd)
|
4702
|
-
|
4703
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4704
|
-
self._readable.discard(fd)
|
4705
|
-
self._poller.unregister(fd)
|
4706
|
-
if fd in self._writable:
|
4707
|
-
self._poller.register(fd, self._WRITE)
|
4708
|
-
|
4709
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4710
|
-
self._writable.discard(fd)
|
4711
|
-
self._poller.unregister(fd)
|
4712
|
-
if fd in self._readable:
|
4713
|
-
self._poller.register(fd, self._READ)
|
4714
|
-
|
4715
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4716
|
-
fds = self._poll_fds(timeout) # type: ignore
|
4717
|
-
readable, writable = [], []
|
4718
|
-
for fd, eventmask in fds:
|
4719
|
-
if self._ignore_invalid(fd, eventmask):
|
4720
|
-
continue
|
4721
|
-
if eventmask & self._READ:
|
4722
|
-
readable.append(fd)
|
4723
|
-
if eventmask & self._WRITE:
|
4724
|
-
writable.append(fd)
|
4725
|
-
return readable, writable
|
4726
|
-
|
4727
|
-
def _poll_fds(self, timeout: float) -> ta.List[ta.Tuple[Fd, Fd]]:
|
4728
|
-
try:
|
4729
|
-
return self._poller.poll(timeout * 1000) # type: ignore
|
4730
|
-
except OSError as exc:
|
4731
|
-
if exc.args[0] == errno.EINTR:
|
4732
|
-
log.debug('EINTR encountered in poll')
|
4733
|
-
return []
|
4734
|
-
raise
|
4735
|
-
|
4736
|
-
def _ignore_invalid(self, fd: Fd, eventmask: int) -> bool:
|
4737
|
-
if eventmask & select.POLLNVAL:
|
4738
|
-
# POLLNVAL means `fd` value is invalid, not open. When a process quits it's `fd`s are closed so there is no
|
4739
|
-
# more reason to keep this `fd` registered If the process restarts it's `fd`s are registered again.
|
4740
|
-
self._poller.unregister(fd)
|
4741
|
-
self._readable.discard(fd)
|
4742
|
-
self._writable.discard(fd)
|
4743
|
-
return True
|
4744
|
-
return False
|
4745
|
-
|
4746
|
-
|
4747
|
-
if sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
|
4748
|
-
class KqueuePoller(Poller):
|
4749
|
-
max_events = 1000
|
4750
|
-
|
4751
|
-
def __init__(self) -> None:
|
4752
|
-
super().__init__()
|
4753
|
-
|
4754
|
-
self._kqueue: ta.Optional[ta.Any] = select.kqueue()
|
4755
|
-
self._readable: set[Fd] = set()
|
4756
|
-
self._writable: set[Fd] = set()
|
4757
|
-
|
4758
|
-
def register_readable(self, fd: Fd) -> None:
|
4759
|
-
self._readable.add(fd)
|
4760
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD)
|
4761
|
-
self._kqueue_control(fd, kevent)
|
4762
|
-
|
4763
|
-
def register_writable(self, fd: Fd) -> None:
|
4764
|
-
self._writable.add(fd)
|
4765
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_ADD)
|
4766
|
-
self._kqueue_control(fd, kevent)
|
4767
|
-
|
4768
|
-
def unregister_readable(self, fd: Fd) -> None:
|
4769
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_DELETE)
|
4770
|
-
self._readable.discard(fd)
|
4771
|
-
self._kqueue_control(fd, kevent)
|
4772
|
-
|
4773
|
-
def unregister_writable(self, fd: Fd) -> None:
|
4774
|
-
kevent = select.kevent(fd, filter=select.KQ_FILTER_WRITE, flags=select.KQ_EV_DELETE)
|
4775
|
-
self._writable.discard(fd)
|
4776
|
-
self._kqueue_control(fd, kevent)
|
4777
|
-
|
4778
|
-
def _kqueue_control(self, fd: Fd, kevent: 'select.kevent') -> None:
|
4779
|
-
try:
|
4780
|
-
self._kqueue.control([kevent], 0) # type: ignore
|
4781
|
-
except OSError as error:
|
4782
|
-
if error.errno == errno.EBADF:
|
4783
|
-
log.debug('EBADF encountered in kqueue. Invalid file descriptor %s', fd)
|
4784
|
-
else:
|
4785
|
-
raise
|
4786
|
-
|
4787
|
-
def poll(self, timeout: ta.Optional[float]) -> ta.Tuple[ta.List[Fd], ta.List[Fd]]:
|
4788
|
-
readable, writable = [], [] # type: ignore
|
4789
|
-
|
4790
|
-
try:
|
4791
|
-
kevents = self._kqueue.control(None, self.max_events, timeout) # type: ignore
|
4792
|
-
except OSError as error:
|
4793
|
-
if error.errno == errno.EINTR:
|
4794
|
-
log.debug('EINTR encountered in poll')
|
4795
|
-
return readable, writable
|
4796
|
-
raise
|
4797
|
-
|
4798
|
-
for kevent in kevents:
|
4799
|
-
if kevent.filter == select.KQ_FILTER_READ:
|
4800
|
-
readable.append(kevent.ident)
|
4801
|
-
if kevent.filter == select.KQ_FILTER_WRITE:
|
4802
|
-
writable.append(kevent.ident)
|
4803
|
-
|
4804
|
-
return readable, writable
|
4805
|
-
|
4806
|
-
def before_daemonize(self) -> None:
|
4807
|
-
self.close()
|
4808
|
-
|
4809
|
-
def after_daemonize(self) -> None:
|
4810
|
-
self._kqueue = select.kqueue()
|
4811
|
-
for fd in self._readable:
|
4812
|
-
self.register_readable(fd)
|
4813
|
-
for fd in self._writable:
|
4814
|
-
self.register_writable(fd)
|
4815
|
-
|
4816
|
-
def close(self) -> None:
|
4817
|
-
self._kqueue.close() # type: ignore
|
4818
|
-
self._kqueue = None
|
4819
|
-
|
4820
|
-
else:
|
4821
|
-
KqueuePoller = None
|
4822
|
-
|
4823
|
-
|
4824
|
-
def get_poller_impl() -> ta.Type[Poller]:
|
4825
|
-
if (
|
4826
|
-
(sys.platform == 'darwin' or sys.platform.startswith('freebsd')) and
|
4827
|
-
hasattr(select, 'kqueue') and
|
4828
|
-
KqueuePoller is not None
|
4829
|
-
):
|
4830
|
-
return KqueuePoller
|
4831
|
-
elif hasattr(select, 'poll'):
|
4832
|
-
return PollPoller
|
4833
|
-
else:
|
4834
|
-
return SelectPoller
|
4835
|
-
|
4836
|
-
|
4837
5339
|
########################################
|
4838
5340
|
# ../../../omlish/lite/http/coroserver.py
|
4839
5341
|
# PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
|
@@ -5442,96 +5944,45 @@ class SupervisorStateManager(abc.ABC):
|
|
5442
5944
|
##
|
5443
5945
|
|
5444
5946
|
|
5445
|
-
class
|
5446
|
-
@property
|
5947
|
+
class HasDispatchers(abc.ABC):
|
5447
5948
|
@abc.abstractmethod
|
5448
|
-
def
|
5949
|
+
def get_dispatchers(self) -> 'Dispatchers':
|
5449
5950
|
raise NotImplementedError
|
5450
5951
|
|
5952
|
+
|
5953
|
+
class ProcessDispatcher(FdIoHandler, abc.ABC):
|
5451
5954
|
@property
|
5452
5955
|
@abc.abstractmethod
|
5453
|
-
def
|
5956
|
+
def channel(self) -> str:
|
5454
5957
|
raise NotImplementedError
|
5455
5958
|
|
5456
5959
|
@property
|
5457
5960
|
@abc.abstractmethod
|
5458
|
-
def
|
5961
|
+
def process(self) -> 'Process':
|
5459
5962
|
raise NotImplementedError
|
5460
5963
|
|
5461
|
-
#
|
5462
5964
|
|
5965
|
+
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
5463
5966
|
@abc.abstractmethod
|
5464
|
-
def
|
5967
|
+
def remove_logs(self) -> None:
|
5465
5968
|
raise NotImplementedError
|
5466
5969
|
|
5467
5970
|
@abc.abstractmethod
|
5468
|
-
def
|
5971
|
+
def reopen_logs(self) -> None:
|
5469
5972
|
raise NotImplementedError
|
5470
5973
|
|
5471
|
-
#
|
5472
5974
|
|
5975
|
+
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
5473
5976
|
@abc.abstractmethod
|
5474
|
-
def
|
5977
|
+
def write(self, chars: ta.Union[bytes, str]) -> None:
|
5475
5978
|
raise NotImplementedError
|
5476
5979
|
|
5477
5980
|
@abc.abstractmethod
|
5478
|
-
def
|
5981
|
+
def flush(self) -> None:
|
5479
5982
|
raise NotImplementedError
|
5480
5983
|
|
5481
|
-
#
|
5482
5984
|
|
5483
|
-
|
5484
|
-
raise TypeError
|
5485
|
-
|
5486
|
-
def handle_write_event(self) -> None:
|
5487
|
-
raise TypeError
|
5488
|
-
|
5489
|
-
#
|
5490
|
-
|
5491
|
-
def handle_connect(self) -> None:
|
5492
|
-
raise TypeError
|
5493
|
-
|
5494
|
-
def handle_close(self) -> None:
|
5495
|
-
raise TypeError
|
5496
|
-
|
5497
|
-
def handle_accepted(self, sock, addr) -> None:
|
5498
|
-
raise TypeError
|
5499
|
-
|
5500
|
-
|
5501
|
-
class HasDispatchers(abc.ABC):
|
5502
|
-
@abc.abstractmethod
|
5503
|
-
def get_dispatchers(self) -> 'Dispatchers':
|
5504
|
-
raise NotImplementedError
|
5505
|
-
|
5506
|
-
|
5507
|
-
class ProcessDispatcher(Dispatcher, abc.ABC):
|
5508
|
-
@property
|
5509
|
-
@abc.abstractmethod
|
5510
|
-
def process(self) -> 'Process':
|
5511
|
-
raise NotImplementedError
|
5512
|
-
|
5513
|
-
|
5514
|
-
class ProcessOutputDispatcher(ProcessDispatcher, abc.ABC):
|
5515
|
-
@abc.abstractmethod
|
5516
|
-
def remove_logs(self) -> None:
|
5517
|
-
raise NotImplementedError
|
5518
|
-
|
5519
|
-
@abc.abstractmethod
|
5520
|
-
def reopen_logs(self) -> None:
|
5521
|
-
raise NotImplementedError
|
5522
|
-
|
5523
|
-
|
5524
|
-
class ProcessInputDispatcher(ProcessDispatcher, abc.ABC):
|
5525
|
-
@abc.abstractmethod
|
5526
|
-
def write(self, chars: ta.Union[bytes, str]) -> None:
|
5527
|
-
raise NotImplementedError
|
5528
|
-
|
5529
|
-
@abc.abstractmethod
|
5530
|
-
def flush(self) -> None:
|
5531
|
-
raise NotImplementedError
|
5532
|
-
|
5533
|
-
|
5534
|
-
##
|
5985
|
+
##
|
5535
5986
|
|
5536
5987
|
|
5537
5988
|
class Process(
|
@@ -5625,13 +6076,139 @@ class ProcessGroup(
|
|
5625
6076
|
raise NotImplementedError
|
5626
6077
|
|
5627
6078
|
|
6079
|
+
########################################
|
6080
|
+
# ../../../omlish/lite/fdio/corohttp.py
|
6081
|
+
|
6082
|
+
|
6083
|
+
class CoroHttpServerConnectionFdIoHandler(SocketFdIoHandler):
|
6084
|
+
def __init__(
|
6085
|
+
self,
|
6086
|
+
addr: SocketAddress,
|
6087
|
+
sock: socket.socket,
|
6088
|
+
handler: HttpHandler,
|
6089
|
+
*,
|
6090
|
+
read_size: int = 0x10000,
|
6091
|
+
write_size: int = 0x10000,
|
6092
|
+
) -> None:
|
6093
|
+
check_state(not sock.getblocking())
|
6094
|
+
|
6095
|
+
super().__init__(addr, sock)
|
6096
|
+
|
6097
|
+
self._handler = handler
|
6098
|
+
self._read_size = read_size
|
6099
|
+
self._write_size = write_size
|
6100
|
+
|
6101
|
+
self._read_buf = ReadableListBuffer()
|
6102
|
+
self._write_buf: IncrementalWriteBuffer | None = None
|
6103
|
+
|
6104
|
+
self._coro_srv = CoroHttpServer(
|
6105
|
+
addr,
|
6106
|
+
handler=self._handler,
|
6107
|
+
)
|
6108
|
+
self._srv_coro: ta.Optional[ta.Generator[CoroHttpServer.Io, ta.Optional[bytes], None]] = self._coro_srv.coro_handle() # noqa
|
6109
|
+
|
6110
|
+
self._cur_io: CoroHttpServer.Io | None = None
|
6111
|
+
self._next_io()
|
6112
|
+
|
6113
|
+
#
|
6114
|
+
|
6115
|
+
def _next_io(self) -> None: # noqa
|
6116
|
+
coro = check_not_none(self._srv_coro)
|
6117
|
+
|
6118
|
+
d: bytes | None = None
|
6119
|
+
o = self._cur_io
|
6120
|
+
while True:
|
6121
|
+
if o is None:
|
6122
|
+
try:
|
6123
|
+
if d is not None:
|
6124
|
+
o = coro.send(d)
|
6125
|
+
d = None
|
6126
|
+
else:
|
6127
|
+
o = next(coro)
|
6128
|
+
except StopIteration:
|
6129
|
+
self.close()
|
6130
|
+
o = None
|
6131
|
+
break
|
6132
|
+
|
6133
|
+
if isinstance(o, CoroHttpServer.AnyLogIo):
|
6134
|
+
print(o)
|
6135
|
+
o = None
|
6136
|
+
|
6137
|
+
elif isinstance(o, CoroHttpServer.ReadIo):
|
6138
|
+
if (d := self._read_buf.read(o.sz)) is None:
|
6139
|
+
break
|
6140
|
+
o = None
|
6141
|
+
|
6142
|
+
elif isinstance(o, CoroHttpServer.ReadLineIo):
|
6143
|
+
if (d := self._read_buf.read_until(b'\n')) is None:
|
6144
|
+
break
|
6145
|
+
o = None
|
6146
|
+
|
6147
|
+
elif isinstance(o, CoroHttpServer.WriteIo):
|
6148
|
+
check_none(self._write_buf)
|
6149
|
+
self._write_buf = IncrementalWriteBuffer(o.data, write_size=self._write_size)
|
6150
|
+
break
|
6151
|
+
|
6152
|
+
else:
|
6153
|
+
raise TypeError(o)
|
6154
|
+
|
6155
|
+
self._cur_io = o
|
6156
|
+
|
6157
|
+
#
|
6158
|
+
|
6159
|
+
def readable(self) -> bool:
|
6160
|
+
return True
|
6161
|
+
|
6162
|
+
def writable(self) -> bool:
|
6163
|
+
return self._write_buf is not None
|
6164
|
+
|
6165
|
+
#
|
6166
|
+
|
6167
|
+
def on_readable(self) -> None:
|
6168
|
+
try:
|
6169
|
+
buf = check_not_none(self._sock).recv(self._read_size)
|
6170
|
+
except BlockingIOError:
|
6171
|
+
return
|
6172
|
+
except ConnectionResetError:
|
6173
|
+
self.close()
|
6174
|
+
return
|
6175
|
+
if not buf:
|
6176
|
+
self.close()
|
6177
|
+
return
|
6178
|
+
|
6179
|
+
self._read_buf.feed(buf)
|
6180
|
+
|
6181
|
+
if isinstance(self._cur_io, CoroHttpServer.AnyReadIo):
|
6182
|
+
self._next_io()
|
6183
|
+
|
6184
|
+
def on_writable(self) -> None:
|
6185
|
+
check_isinstance(self._cur_io, CoroHttpServer.WriteIo)
|
6186
|
+
wb = check_not_none(self._write_buf)
|
6187
|
+
while wb.rem > 0:
|
6188
|
+
def send(d: bytes) -> int:
|
6189
|
+
try:
|
6190
|
+
return check_not_none(self._sock).send(d)
|
6191
|
+
except ConnectionResetError:
|
6192
|
+
self.close()
|
6193
|
+
return 0
|
6194
|
+
except BlockingIOError:
|
6195
|
+
return 0
|
6196
|
+
if not wb.write(send):
|
6197
|
+
break
|
6198
|
+
|
6199
|
+
if wb.rem < 1:
|
6200
|
+
self._write_buf = None
|
6201
|
+
self._cur_io = None
|
6202
|
+
self._next_io()
|
6203
|
+
|
6204
|
+
|
5628
6205
|
########################################
|
5629
6206
|
# ../dispatchers.py
|
5630
6207
|
|
5631
6208
|
|
5632
|
-
class Dispatchers(KeyedCollection[Fd,
|
5633
|
-
def _key(self, v:
|
5634
|
-
return v.fd
|
6209
|
+
class Dispatchers(KeyedCollection[Fd, FdIoHandler]):
|
6210
|
+
def _key(self, v: FdIoHandler) -> Fd:
|
6211
|
+
return Fd(v.fd())
|
5635
6212
|
|
5636
6213
|
#
|
5637
6214
|
|
@@ -5640,9 +6217,9 @@ class Dispatchers(KeyedCollection[Fd, Dispatcher]):
|
|
5640
6217
|
# note that we *must* call readable() for every dispatcher, as it may have side effects for a given
|
5641
6218
|
# dispatcher (eg. call handle_listener_state_change for event listener processes)
|
5642
6219
|
if d.readable():
|
5643
|
-
d.
|
6220
|
+
d.on_readable()
|
5644
6221
|
if d.writable():
|
5645
|
-
d.
|
6222
|
+
d.on_writable()
|
5646
6223
|
|
5647
6224
|
#
|
5648
6225
|
|
@@ -5696,7 +6273,6 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
5696
6273
|
def channel(self) -> str:
|
5697
6274
|
return self._channel
|
5698
6275
|
|
5699
|
-
@property
|
5700
6276
|
def fd(self) -> Fd:
|
5701
6277
|
return self._fd
|
5702
6278
|
|
@@ -5711,7 +6287,7 @@ class BaseProcessDispatcherImpl(ProcessDispatcher, abc.ABC):
|
|
5711
6287
|
log.debug('fd %s closed, stopped monitoring %s', self._fd, self)
|
5712
6288
|
self._closed = True
|
5713
6289
|
|
5714
|
-
def
|
6290
|
+
def on_error(self, exc: ta.Optional[BaseException] = None) -> None:
|
5715
6291
|
nil, t, v, tbinfo = compact_traceback()
|
5716
6292
|
|
5717
6293
|
log.critical('uncaptured python exception, closing channel %s (%s:%s %s)', repr(self), t, v, tbinfo)
|
@@ -5927,7 +6503,7 @@ class ProcessOutputDispatcherImpl(BaseProcessDispatcherImpl, ProcessOutputDispat
|
|
5927
6503
|
return False
|
5928
6504
|
return True
|
5929
6505
|
|
5930
|
-
def
|
6506
|
+
def on_readable(self) -> None:
|
5931
6507
|
data = read_fd(self._fd)
|
5932
6508
|
self._output_buffer += data
|
5933
6509
|
self.record_output()
|
@@ -5965,15 +6541,12 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
|
|
5965
6541
|
return True
|
5966
6542
|
return False
|
5967
6543
|
|
5968
|
-
def readable(self) -> bool:
|
5969
|
-
return False
|
5970
|
-
|
5971
6544
|
def flush(self) -> None:
|
5972
6545
|
# other code depends on this raising EPIPE if the pipe is closed
|
5973
6546
|
sent = os.write(self._fd, as_bytes(self._input_buffer))
|
5974
6547
|
self._input_buffer = self._input_buffer[sent:]
|
5975
6548
|
|
5976
|
-
def
|
6549
|
+
def on_writable(self) -> None:
|
5977
6550
|
if self._input_buffer:
|
5978
6551
|
try:
|
5979
6552
|
self.flush()
|
@@ -5985,79 +6558,6 @@ class ProcessInputDispatcherImpl(BaseProcessDispatcherImpl, ProcessInputDispatch
|
|
5985
6558
|
raise
|
5986
6559
|
|
5987
6560
|
|
5988
|
-
########################################
|
5989
|
-
# ../groups.py
|
5990
|
-
|
5991
|
-
|
5992
|
-
class ProcessGroupManager(KeyedCollectionAccessors[str, ProcessGroup]):
|
5993
|
-
def __init__(
|
5994
|
-
self,
|
5995
|
-
*,
|
5996
|
-
event_callbacks: EventCallbacks,
|
5997
|
-
) -> None:
|
5998
|
-
super().__init__()
|
5999
|
-
|
6000
|
-
self._event_callbacks = event_callbacks
|
6001
|
-
|
6002
|
-
self._by_name: ta.Dict[str, ProcessGroup] = {}
|
6003
|
-
|
6004
|
-
@property
|
6005
|
-
def _by_key(self) -> ta.Mapping[str, ProcessGroup]:
|
6006
|
-
return self._by_name
|
6007
|
-
|
6008
|
-
#
|
6009
|
-
|
6010
|
-
def all_processes(self) -> ta.Iterator[Process]:
|
6011
|
-
for g in self:
|
6012
|
-
yield from g
|
6013
|
-
|
6014
|
-
#
|
6015
|
-
|
6016
|
-
def add(self, group: ProcessGroup) -> None:
|
6017
|
-
if (name := group.name) in self._by_name:
|
6018
|
-
raise KeyError(f'Process group already exists: {name}')
|
6019
|
-
|
6020
|
-
self._by_name[name] = group
|
6021
|
-
|
6022
|
-
self._event_callbacks.notify(ProcessGroupAddedEvent(name))
|
6023
|
-
|
6024
|
-
def remove(self, name: str) -> None:
|
6025
|
-
group = self._by_name[name]
|
6026
|
-
|
6027
|
-
group.before_remove()
|
6028
|
-
|
6029
|
-
del self._by_name[name]
|
6030
|
-
|
6031
|
-
self._event_callbacks.notify(ProcessGroupRemovedEvent(name))
|
6032
|
-
|
6033
|
-
def clear(self) -> None:
|
6034
|
-
# FIXME: events?
|
6035
|
-
self._by_name.clear()
|
6036
|
-
|
6037
|
-
#
|
6038
|
-
|
6039
|
-
class Diff(ta.NamedTuple):
|
6040
|
-
added: ta.List[ProcessGroupConfig]
|
6041
|
-
changed: ta.List[ProcessGroupConfig]
|
6042
|
-
removed: ta.List[ProcessGroupConfig]
|
6043
|
-
|
6044
|
-
def diff(self, new: ta.Sequence[ProcessGroupConfig]) -> Diff:
|
6045
|
-
cur = [group.config for group in self]
|
6046
|
-
|
6047
|
-
cur_by_name = {cfg.name: cfg for cfg in cur}
|
6048
|
-
new_by_name = {cfg.name: cfg for cfg in new}
|
6049
|
-
|
6050
|
-
added = [cand for cand in new if cand.name not in cur_by_name]
|
6051
|
-
removed = [cand for cand in cur if cand.name not in new_by_name]
|
6052
|
-
changed = [cand for cand in new if cand != cur_by_name.get(cand.name, cand)]
|
6053
|
-
|
6054
|
-
return ProcessGroupManager.Diff(
|
6055
|
-
added,
|
6056
|
-
changed,
|
6057
|
-
removed,
|
6058
|
-
)
|
6059
|
-
|
6060
|
-
|
6061
6561
|
########################################
|
6062
6562
|
# ../groupsimpl.py
|
6063
6563
|
|
@@ -6402,6 +6902,92 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
6402
6902
|
os.umask(self._config.umask)
|
6403
6903
|
|
6404
6904
|
|
6905
|
+
########################################
|
6906
|
+
# ../groups.py
|
6907
|
+
|
6908
|
+
|
6909
|
+
class ProcessGroupManager(
|
6910
|
+
KeyedCollectionAccessors[str, ProcessGroup],
|
6911
|
+
HasDispatchers,
|
6912
|
+
):
|
6913
|
+
def __init__(
|
6914
|
+
self,
|
6915
|
+
*,
|
6916
|
+
event_callbacks: EventCallbacks,
|
6917
|
+
) -> None:
|
6918
|
+
super().__init__()
|
6919
|
+
|
6920
|
+
self._event_callbacks = event_callbacks
|
6921
|
+
|
6922
|
+
self._by_name: ta.Dict[str, ProcessGroup] = {}
|
6923
|
+
|
6924
|
+
@property
|
6925
|
+
def _by_key(self) -> ta.Mapping[str, ProcessGroup]:
|
6926
|
+
return self._by_name
|
6927
|
+
|
6928
|
+
#
|
6929
|
+
|
6930
|
+
def all_processes(self) -> ta.Iterator[Process]:
|
6931
|
+
for g in self:
|
6932
|
+
yield from g
|
6933
|
+
|
6934
|
+
#
|
6935
|
+
|
6936
|
+
def get_dispatchers(self) -> Dispatchers:
|
6937
|
+
return Dispatchers(
|
6938
|
+
d
|
6939
|
+
for g in self
|
6940
|
+
for p in g
|
6941
|
+
for d in p.get_dispatchers()
|
6942
|
+
)
|
6943
|
+
|
6944
|
+
#
|
6945
|
+
|
6946
|
+
def add(self, group: ProcessGroup) -> None:
|
6947
|
+
if (name := group.name) in self._by_name:
|
6948
|
+
raise KeyError(f'Process group already exists: {name}')
|
6949
|
+
|
6950
|
+
self._by_name[name] = group
|
6951
|
+
|
6952
|
+
self._event_callbacks.notify(ProcessGroupAddedEvent(name))
|
6953
|
+
|
6954
|
+
def remove(self, name: str) -> None:
|
6955
|
+
group = self._by_name[name]
|
6956
|
+
|
6957
|
+
group.before_remove()
|
6958
|
+
|
6959
|
+
del self._by_name[name]
|
6960
|
+
|
6961
|
+
self._event_callbacks.notify(ProcessGroupRemovedEvent(name))
|
6962
|
+
|
6963
|
+
def clear(self) -> None:
|
6964
|
+
# FIXME: events?
|
6965
|
+
self._by_name.clear()
|
6966
|
+
|
6967
|
+
#
|
6968
|
+
|
6969
|
+
class Diff(ta.NamedTuple):
|
6970
|
+
added: ta.List[ProcessGroupConfig]
|
6971
|
+
changed: ta.List[ProcessGroupConfig]
|
6972
|
+
removed: ta.List[ProcessGroupConfig]
|
6973
|
+
|
6974
|
+
def diff(self, new: ta.Sequence[ProcessGroupConfig]) -> Diff:
|
6975
|
+
cur = [group.config for group in self]
|
6976
|
+
|
6977
|
+
cur_by_name = {cfg.name: cfg for cfg in cur}
|
6978
|
+
new_by_name = {cfg.name: cfg for cfg in new}
|
6979
|
+
|
6980
|
+
added = [cand for cand in new if cand.name not in cur_by_name]
|
6981
|
+
removed = [cand for cand in cur if cand.name not in new_by_name]
|
6982
|
+
changed = [cand for cand in new if cand != cur_by_name.get(cand.name, cand)]
|
6983
|
+
|
6984
|
+
return ProcessGroupManager.Diff(
|
6985
|
+
added,
|
6986
|
+
changed,
|
6987
|
+
removed,
|
6988
|
+
)
|
6989
|
+
|
6990
|
+
|
6405
6991
|
########################################
|
6406
6992
|
# ../io.py
|
6407
6993
|
|
@@ -6409,49 +6995,59 @@ class SupervisorSetupImpl(SupervisorSetup):
|
|
6409
6995
|
##
|
6410
6996
|
|
6411
6997
|
|
6412
|
-
|
6998
|
+
HasDispatchersList = ta.NewType('HasDispatchersList', ta.Sequence[HasDispatchers])
|
6999
|
+
|
7000
|
+
|
7001
|
+
class IoManager(HasDispatchers):
|
6413
7002
|
def __init__(
|
6414
7003
|
self,
|
6415
7004
|
*,
|
6416
|
-
poller:
|
6417
|
-
|
7005
|
+
poller: FdIoPoller,
|
7006
|
+
has_dispatchers_list: HasDispatchersList,
|
6418
7007
|
) -> None:
|
6419
7008
|
super().__init__()
|
6420
7009
|
|
6421
7010
|
self._poller = poller
|
6422
|
-
self.
|
7011
|
+
self._has_dispatchers_list = has_dispatchers_list
|
6423
7012
|
|
6424
7013
|
def get_dispatchers(self) -> Dispatchers:
|
6425
7014
|
return Dispatchers(
|
6426
7015
|
d
|
6427
|
-
for
|
6428
|
-
for d in
|
7016
|
+
for hd in self._has_dispatchers_list
|
7017
|
+
for d in hd.get_dispatchers()
|
6429
7018
|
)
|
6430
7019
|
|
6431
7020
|
def poll(self) -> None:
|
6432
7021
|
dispatchers = self.get_dispatchers()
|
6433
7022
|
|
6434
|
-
|
6435
|
-
if
|
6436
|
-
|
6437
|
-
|
6438
|
-
self._poller.register_writable(fd)
|
7023
|
+
self._poller.update(
|
7024
|
+
{fd for fd, d in dispatchers.items() if d.readable()},
|
7025
|
+
{fd for fd, d in dispatchers.items() if d.writable()},
|
7026
|
+
)
|
6439
7027
|
|
6440
7028
|
timeout = 1 # this cannot be fewer than the smallest TickEvent (5)
|
6441
|
-
|
6442
|
-
|
6443
|
-
|
7029
|
+
log.info(f'Polling: {timeout=}') # noqa
|
7030
|
+
polled = self._poller.poll(timeout)
|
7031
|
+
log.info(f'Polled: {polled=}') # noqa
|
7032
|
+
if polled.msg is not None:
|
7033
|
+
log.error(polled.msg)
|
7034
|
+
if polled.exc is not None:
|
7035
|
+
log.error('Poll exception: %r', polled.exc)
|
7036
|
+
|
7037
|
+
for r in polled.r:
|
7038
|
+
fd = Fd(r)
|
6444
7039
|
if fd in dispatchers:
|
7040
|
+
dispatcher = dispatchers[fd]
|
6445
7041
|
try:
|
6446
|
-
dispatcher = dispatchers[fd]
|
6447
7042
|
log.debug('read event caused by %r', dispatcher)
|
6448
|
-
dispatcher.
|
7043
|
+
dispatcher.on_readable()
|
6449
7044
|
if not dispatcher.readable():
|
6450
7045
|
self._poller.unregister_readable(fd)
|
6451
7046
|
except ExitNow:
|
6452
7047
|
raise
|
6453
|
-
except Exception: # noqa
|
6454
|
-
|
7048
|
+
except Exception as exc: # noqa
|
7049
|
+
log.exception('Error in dispatcher: %r', dispatcher)
|
7050
|
+
dispatcher.on_error(exc)
|
6455
7051
|
else:
|
6456
7052
|
# if the fd is not in combined map, we should unregister it. otherwise, it will be polled every
|
6457
7053
|
# time, which may cause 100% cpu usage
|
@@ -6461,18 +7057,20 @@ class IoManager:
|
|
6461
7057
|
except Exception: # noqa
|
6462
7058
|
pass
|
6463
7059
|
|
6464
|
-
for
|
7060
|
+
for w in polled.w:
|
7061
|
+
fd = Fd(w)
|
6465
7062
|
if fd in dispatchers:
|
7063
|
+
dispatcher = dispatchers[fd]
|
6466
7064
|
try:
|
6467
|
-
dispatcher = dispatchers[fd]
|
6468
7065
|
log.debug('write event caused by %r', dispatcher)
|
6469
|
-
dispatcher.
|
7066
|
+
dispatcher.on_writable()
|
6470
7067
|
if not dispatcher.writable():
|
6471
7068
|
self._poller.unregister_writable(fd)
|
6472
7069
|
except ExitNow:
|
6473
7070
|
raise
|
6474
|
-
except Exception: # noqa
|
6475
|
-
|
7071
|
+
except Exception as exc: # noqa
|
7072
|
+
log.exception('Error in dispatcher: %r', dispatcher)
|
7073
|
+
dispatcher.on_error(exc)
|
6476
7074
|
else:
|
6477
7075
|
log.debug('unexpected write event from fd %r', fd)
|
6478
7076
|
try:
|
@@ -6481,65 +7079,6 @@ class IoManager:
|
|
6481
7079
|
pass
|
6482
7080
|
|
6483
7081
|
|
6484
|
-
########################################
|
6485
|
-
# ../signals.py
|
6486
|
-
|
6487
|
-
|
6488
|
-
class SignalHandler:
|
6489
|
-
def __init__(
|
6490
|
-
self,
|
6491
|
-
*,
|
6492
|
-
states: SupervisorStateManager,
|
6493
|
-
signal_receiver: SignalReceiver,
|
6494
|
-
process_groups: ProcessGroupManager,
|
6495
|
-
) -> None:
|
6496
|
-
super().__init__()
|
6497
|
-
|
6498
|
-
self._states = states
|
6499
|
-
self._signal_receiver = signal_receiver
|
6500
|
-
self._process_groups = process_groups
|
6501
|
-
|
6502
|
-
def set_signals(self) -> None:
|
6503
|
-
self._signal_receiver.install(
|
6504
|
-
signal.SIGTERM,
|
6505
|
-
signal.SIGINT,
|
6506
|
-
signal.SIGQUIT,
|
6507
|
-
signal.SIGHUP,
|
6508
|
-
signal.SIGCHLD,
|
6509
|
-
signal.SIGUSR2,
|
6510
|
-
)
|
6511
|
-
|
6512
|
-
def handle_signals(self) -> None:
|
6513
|
-
sig = self._signal_receiver.get_signal()
|
6514
|
-
if not sig:
|
6515
|
-
return
|
6516
|
-
|
6517
|
-
if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
|
6518
|
-
log.warning('received %s indicating exit request', sig_name(sig))
|
6519
|
-
self._states.set_state(SupervisorState.SHUTDOWN)
|
6520
|
-
|
6521
|
-
elif sig == signal.SIGHUP:
|
6522
|
-
if self._states.state == SupervisorState.SHUTDOWN:
|
6523
|
-
log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig)) # noqa
|
6524
|
-
else:
|
6525
|
-
log.warning('received %s indicating restart request', sig_name(sig)) # noqa
|
6526
|
-
self._states.set_state(SupervisorState.RESTARTING)
|
6527
|
-
|
6528
|
-
elif sig == signal.SIGCHLD:
|
6529
|
-
log.debug('received %s indicating a child quit', sig_name(sig))
|
6530
|
-
|
6531
|
-
elif sig == signal.SIGUSR2:
|
6532
|
-
log.info('received %s indicating log reopen request', sig_name(sig))
|
6533
|
-
|
6534
|
-
for p in self._process_groups.all_processes():
|
6535
|
-
for d in p.get_dispatchers():
|
6536
|
-
if isinstance(d, ProcessOutputDispatcher):
|
6537
|
-
d.reopen_logs()
|
6538
|
-
|
6539
|
-
else:
|
6540
|
-
log.debug('received %s indicating nothing', sig_name(sig))
|
6541
|
-
|
6542
|
-
|
6543
7082
|
########################################
|
6544
7083
|
# ../spawning.py
|
6545
7084
|
|
@@ -6568,6 +7107,123 @@ class ProcessSpawning:
|
|
6568
7107
|
raise NotImplementedError
|
6569
7108
|
|
6570
7109
|
|
7110
|
+
########################################
|
7111
|
+
# ../http.py
|
7112
|
+
|
7113
|
+
|
7114
|
+
##
|
7115
|
+
|
7116
|
+
|
7117
|
+
class SocketServerFdIoHandler(SocketFdIoHandler):
|
7118
|
+
def __init__(
|
7119
|
+
self,
|
7120
|
+
addr: SocketAddress,
|
7121
|
+
on_connect: ta.Callable[[socket.socket, SocketAddress], None],
|
7122
|
+
) -> None:
|
7123
|
+
sock = socket.create_server(addr)
|
7124
|
+
sock.setblocking(False)
|
7125
|
+
|
7126
|
+
super().__init__(addr, sock)
|
7127
|
+
|
7128
|
+
self._on_connect = on_connect
|
7129
|
+
|
7130
|
+
sock.listen(1)
|
7131
|
+
|
7132
|
+
def readable(self) -> bool:
|
7133
|
+
return True
|
7134
|
+
|
7135
|
+
def on_readable(self) -> None:
|
7136
|
+
cli_sock, cli_addr = check_not_none(self._sock).accept()
|
7137
|
+
cli_sock.setblocking(False)
|
7138
|
+
|
7139
|
+
self._on_connect(cli_sock, cli_addr)
|
7140
|
+
|
7141
|
+
|
7142
|
+
##
|
7143
|
+
|
7144
|
+
|
7145
|
+
class HttpServer(HasDispatchers):
|
7146
|
+
class Address(ta.NamedTuple):
|
7147
|
+
a: SocketAddress
|
7148
|
+
|
7149
|
+
class Handler(ta.NamedTuple):
|
7150
|
+
h: HttpHandler
|
7151
|
+
|
7152
|
+
def __init__(
|
7153
|
+
self,
|
7154
|
+
handler: Handler,
|
7155
|
+
addr: Address = Address(('localhost', 8000)),
|
7156
|
+
) -> None:
|
7157
|
+
super().__init__()
|
7158
|
+
|
7159
|
+
self._handler = handler.h
|
7160
|
+
self._addr = addr.a
|
7161
|
+
|
7162
|
+
self._server = SocketServerFdIoHandler(self._addr, self._on_connect)
|
7163
|
+
|
7164
|
+
self._conns: ta.List[CoroHttpServerConnectionFdIoHandler] = []
|
7165
|
+
|
7166
|
+
def get_dispatchers(self) -> Dispatchers:
|
7167
|
+
l = []
|
7168
|
+
for c in self._conns:
|
7169
|
+
if not c.closed:
|
7170
|
+
l.append(c)
|
7171
|
+
self._conns = l
|
7172
|
+
return Dispatchers([
|
7173
|
+
self._server,
|
7174
|
+
*l,
|
7175
|
+
])
|
7176
|
+
|
7177
|
+
def _on_connect(self, sock: socket.socket, addr: SocketAddress) -> None:
|
7178
|
+
conn = CoroHttpServerConnectionFdIoHandler(
|
7179
|
+
addr,
|
7180
|
+
sock,
|
7181
|
+
self._handler,
|
7182
|
+
)
|
7183
|
+
|
7184
|
+
self._conns.append(conn)
|
7185
|
+
|
7186
|
+
|
7187
|
+
##
|
7188
|
+
|
7189
|
+
|
7190
|
+
class SupervisorHttpHandler:
|
7191
|
+
def __init__(
|
7192
|
+
self,
|
7193
|
+
*,
|
7194
|
+
groups: ProcessGroupManager,
|
7195
|
+
) -> None:
|
7196
|
+
super().__init__()
|
7197
|
+
|
7198
|
+
self._groups = groups
|
7199
|
+
|
7200
|
+
def handle(self, req: HttpHandlerRequest) -> HttpHandlerResponse:
|
7201
|
+
dct = {
|
7202
|
+
'method': req.method,
|
7203
|
+
'path': req.path,
|
7204
|
+
'data': len(req.data or b''),
|
7205
|
+
'groups': {
|
7206
|
+
g.name: {
|
7207
|
+
'processes': {
|
7208
|
+
p.name: {
|
7209
|
+
'pid': p.pid,
|
7210
|
+
}
|
7211
|
+
for p in g
|
7212
|
+
},
|
7213
|
+
}
|
7214
|
+
for g in self._groups
|
7215
|
+
},
|
7216
|
+
}
|
7217
|
+
|
7218
|
+
return HttpHandlerResponse(
|
7219
|
+
200,
|
7220
|
+
data=json.dumps(dct, **JSON_PRETTY_KWARGS).encode('utf-8') + b'\n',
|
7221
|
+
headers={
|
7222
|
+
'Content-Type': 'application/json',
|
7223
|
+
},
|
7224
|
+
)
|
7225
|
+
|
7226
|
+
|
6571
7227
|
########################################
|
6572
7228
|
# ../processimpl.py
|
6573
7229
|
|
@@ -6918,6 +7574,7 @@ class ProcessImpl(Process):
|
|
6918
7574
|
self._last_stop = now
|
6919
7575
|
|
6920
7576
|
if now > self._last_start:
|
7577
|
+
log.info(f'{now - self._last_start=}') # noqa
|
6921
7578
|
too_quickly = now - self._last_start < self._config.startsecs
|
6922
7579
|
else:
|
6923
7580
|
too_quickly = False
|
@@ -6987,8 +7644,6 @@ class ProcessImpl(Process):
|
|
6987
7644
|
|
6988
7645
|
self._check_and_adjust_for_system_clock_rollback(now)
|
6989
7646
|
|
6990
|
-
logger = log
|
6991
|
-
|
6992
7647
|
if self._supervisor_states.state > SupervisorState.RESTARTING:
|
6993
7648
|
# dont start any processes if supervisor is shutting down
|
6994
7649
|
if state == ProcessState.EXITED:
|
@@ -7020,14 +7675,14 @@ class ProcessImpl(Process):
|
|
7020
7675
|
self.check_in_state(ProcessState.STARTING)
|
7021
7676
|
self.change_state(ProcessState.RUNNING)
|
7022
7677
|
msg = ('entered RUNNING state, process has stayed up for > than %s seconds (startsecs)' % self._config.startsecs) # noqa
|
7023
|
-
|
7678
|
+
log.info('success: %s %s', self.name, msg)
|
7024
7679
|
|
7025
7680
|
if state == ProcessState.BACKOFF:
|
7026
7681
|
if self._backoff > self._config.startretries:
|
7027
7682
|
# BACKOFF -> FATAL if the proc has exceeded its number of retries
|
7028
7683
|
self.give_up()
|
7029
7684
|
msg = ('entered FATAL state, too many start retries too quickly')
|
7030
|
-
|
7685
|
+
log.info('gave up: %s %s', self.name, msg)
|
7031
7686
|
|
7032
7687
|
elif state == ProcessState.STOPPING:
|
7033
7688
|
time_left = self._delay - now
|
@@ -7049,6 +7704,65 @@ class ProcessImpl(Process):
|
|
7049
7704
|
pass
|
7050
7705
|
|
7051
7706
|
|
7707
|
+
########################################
|
7708
|
+
# ../signals.py
|
7709
|
+
|
7710
|
+
|
7711
|
+
class SignalHandler:
|
7712
|
+
def __init__(
|
7713
|
+
self,
|
7714
|
+
*,
|
7715
|
+
states: SupervisorStateManager,
|
7716
|
+
signal_receiver: SignalReceiver,
|
7717
|
+
process_groups: ProcessGroupManager,
|
7718
|
+
) -> None:
|
7719
|
+
super().__init__()
|
7720
|
+
|
7721
|
+
self._states = states
|
7722
|
+
self._signal_receiver = signal_receiver
|
7723
|
+
self._process_groups = process_groups
|
7724
|
+
|
7725
|
+
def set_signals(self) -> None:
|
7726
|
+
self._signal_receiver.install(
|
7727
|
+
signal.SIGTERM,
|
7728
|
+
signal.SIGINT,
|
7729
|
+
signal.SIGQUIT,
|
7730
|
+
signal.SIGHUP,
|
7731
|
+
signal.SIGCHLD,
|
7732
|
+
signal.SIGUSR2,
|
7733
|
+
)
|
7734
|
+
|
7735
|
+
def handle_signals(self) -> None:
|
7736
|
+
sig = self._signal_receiver.get_signal()
|
7737
|
+
if not sig:
|
7738
|
+
return
|
7739
|
+
|
7740
|
+
if sig in (signal.SIGTERM, signal.SIGINT, signal.SIGQUIT):
|
7741
|
+
log.warning('received %s indicating exit request', sig_name(sig))
|
7742
|
+
self._states.set_state(SupervisorState.SHUTDOWN)
|
7743
|
+
|
7744
|
+
elif sig == signal.SIGHUP:
|
7745
|
+
if self._states.state == SupervisorState.SHUTDOWN:
|
7746
|
+
log.warning('ignored %s indicating restart request (shutdown in progress)', sig_name(sig)) # noqa
|
7747
|
+
else:
|
7748
|
+
log.warning('received %s indicating restart request', sig_name(sig)) # noqa
|
7749
|
+
self._states.set_state(SupervisorState.RESTARTING)
|
7750
|
+
|
7751
|
+
elif sig == signal.SIGCHLD:
|
7752
|
+
log.debug('received %s indicating a child quit', sig_name(sig))
|
7753
|
+
|
7754
|
+
elif sig == signal.SIGUSR2:
|
7755
|
+
log.info('received %s indicating log reopen request', sig_name(sig))
|
7756
|
+
|
7757
|
+
for p in self._process_groups.all_processes():
|
7758
|
+
for d in p.get_dispatchers():
|
7759
|
+
if isinstance(d, ProcessOutputDispatcher):
|
7760
|
+
d.reopen_logs()
|
7761
|
+
|
7762
|
+
else:
|
7763
|
+
log.debug('received %s indicating nothing', sig_name(sig))
|
7764
|
+
|
7765
|
+
|
7052
7766
|
########################################
|
7053
7767
|
# ../spawningimpl.py
|
7054
7768
|
|
@@ -7209,7 +7923,7 @@ class ProcessSpawningImpl(ProcessSpawning):
|
|
7209
7923
|
return exe, args
|
7210
7924
|
|
7211
7925
|
def _make_dispatchers(self, pipes: ProcessPipes) -> Dispatchers:
|
7212
|
-
dispatchers: ta.List[
|
7926
|
+
dispatchers: ta.List[FdIoHandler] = []
|
7213
7927
|
|
7214
7928
|
if pipes.stdout is not None:
|
7215
7929
|
dispatchers.append(check_isinstance(self._output_dispatcher_factory(
|
@@ -7399,7 +8113,7 @@ class Supervisor:
|
|
7399
8113
|
self,
|
7400
8114
|
*,
|
7401
8115
|
config: ServerConfig,
|
7402
|
-
poller:
|
8116
|
+
poller: FdIoPoller,
|
7403
8117
|
process_groups: ProcessGroupManager,
|
7404
8118
|
signal_handler: SignalHandler,
|
7405
8119
|
event_callbacks: EventCallbacks,
|
@@ -7515,7 +8229,9 @@ class Supervisor:
|
|
7515
8229
|
#
|
7516
8230
|
|
7517
8231
|
def _run_once(self) -> None:
|
8232
|
+
now = time.time()
|
7518
8233
|
self._poll()
|
8234
|
+
log.info(f'Poll took {time.time() - now}') # noqa
|
7519
8235
|
self._reap()
|
7520
8236
|
self._signal_handler.handle_signals()
|
7521
8237
|
self._tick()
|
@@ -7567,6 +8283,7 @@ class Supervisor:
|
|
7567
8283
|
return
|
7568
8284
|
|
7569
8285
|
wp = waitpid()
|
8286
|
+
log.info(f'Waited pid: {wp}') # noqa
|
7570
8287
|
if wp is None or not wp.pid:
|
7571
8288
|
return
|
7572
8289
|
|
@@ -7635,6 +8352,17 @@ def waitpid() -> ta.Optional[WaitedPid]:
|
|
7635
8352
|
# ../inject.py
|
7636
8353
|
|
7637
8354
|
|
8355
|
+
@dc.dataclass(frozen=True)
|
8356
|
+
class _FdIoPollerDaemonizeListener(DaemonizeListener):
|
8357
|
+
_poller: FdIoPoller
|
8358
|
+
|
8359
|
+
def before_daemonize(self) -> None:
|
8360
|
+
self._poller.close()
|
8361
|
+
|
8362
|
+
def after_daemonize(self) -> None:
|
8363
|
+
self._poller.reopen()
|
8364
|
+
|
8365
|
+
|
7638
8366
|
def bind_server(
|
7639
8367
|
config: ServerConfig,
|
7640
8368
|
*,
|
@@ -7644,22 +8372,24 @@ def bind_server(
|
|
7644
8372
|
lst: ta.List[InjectorBindingOrBindings] = [
|
7645
8373
|
inj.bind(config),
|
7646
8374
|
|
8375
|
+
inj.bind_array(DaemonizeListener),
|
7647
8376
|
inj.bind_array_type(DaemonizeListener, DaemonizeListeners),
|
7648
8377
|
|
7649
8378
|
inj.bind(SupervisorSetupImpl, singleton=True),
|
7650
8379
|
inj.bind(SupervisorSetup, to_key=SupervisorSetupImpl),
|
7651
8380
|
|
7652
|
-
inj.bind(DaemonizeListener, array=True, to_key=Poller),
|
7653
|
-
|
7654
8381
|
inj.bind(EventCallbacks, singleton=True),
|
7655
8382
|
|
7656
8383
|
inj.bind(SignalReceiver, singleton=True),
|
7657
8384
|
|
7658
8385
|
inj.bind(IoManager, singleton=True),
|
8386
|
+
inj.bind_array(HasDispatchers),
|
8387
|
+
inj.bind_array_type(HasDispatchers, HasDispatchersList),
|
7659
8388
|
|
7660
8389
|
inj.bind(SignalHandler, singleton=True),
|
7661
8390
|
|
7662
8391
|
inj.bind(ProcessGroupManager, singleton=True),
|
8392
|
+
inj.bind(HasDispatchers, array=True, to_key=ProcessGroupManager),
|
7663
8393
|
|
7664
8394
|
inj.bind(Supervisor, singleton=True),
|
7665
8395
|
|
@@ -7692,7 +8422,26 @@ def bind_server(
|
|
7692
8422
|
|
7693
8423
|
#
|
7694
8424
|
|
7695
|
-
|
8425
|
+
poller_impl = next(filter(None, [
|
8426
|
+
KqueueFdIoPoller,
|
8427
|
+
PollFdIoPoller,
|
8428
|
+
SelectFdIoPoller,
|
8429
|
+
]))
|
8430
|
+
lst.append(inj.bind(poller_impl, key=FdIoPoller, singleton=True))
|
8431
|
+
inj.bind(_FdIoPollerDaemonizeListener, array=True, singleton=True)
|
8432
|
+
|
8433
|
+
#
|
8434
|
+
|
8435
|
+
def _provide_http_handler(s: SupervisorHttpHandler) -> HttpServer.Handler:
|
8436
|
+
return HttpServer.Handler(s.handle)
|
8437
|
+
|
8438
|
+
lst.extend([
|
8439
|
+
inj.bind(HttpServer, singleton=True, eager=True),
|
8440
|
+
inj.bind(HasDispatchers, array=True, to_key=HttpServer),
|
8441
|
+
|
8442
|
+
inj.bind(SupervisorHttpHandler, singleton=True),
|
8443
|
+
inj.bind(_provide_http_handler),
|
8444
|
+
])
|
7696
8445
|
|
7697
8446
|
#
|
7698
8447
|
|