omlish 0.0.0.dev14__tar.gz → 0.0.0.dev15__tar.gz

{omlish-0.0.0.dev14/omlish.egg-info → omlish-0.0.0.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev14
+Version: 0.0.0.dev15
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -22,10 +22,10 @@ Requires-Dist: lz4~=4.0; extra == "all"
 Requires-Dist: python-snappy~=0.7; python_version < "3.13" and extra == "all"
 Requires-Dist: zstd~=1.5; extra == "all"
 Requires-Dist: psutil~=6.0; extra == "all"
-Requires-Dist: cloudpickle~=3.0; extra == "all"
+Requires-Dist: orjson~=3.10; extra == "all"
 Requires-Dist: json5~=0.9; extra == "all"
-Requires-Dist: orjson>3.10; extra == "all"
 Requires-Dist: pyyaml~=5.0; extra == "all"
+Requires-Dist: cloudpickle~=3.0; extra == "all"
 Requires-Dist: httpx[http2]~=0.27; extra == "all"
 Requires-Dist: jinja2~=3.1; extra == "all"
 Requires-Dist: wrapt~=1.14; extra == "all"
@@ -53,10 +53,10 @@ Requires-Dist: zstd~=1.5; extra == "compression"
 Provides-Extra: diag
 Requires-Dist: psutil~=6.0; extra == "diag"
 Provides-Extra: formats
-Requires-Dist: cloudpickle~=3.0; extra == "formats"
+Requires-Dist: orjson~=3.10; extra == "formats"
 Requires-Dist: json5~=0.9; extra == "formats"
-Requires-Dist: orjson>3.10; extra == "formats"
 Requires-Dist: pyyaml~=5.0; extra == "formats"
+Requires-Dist: cloudpickle~=3.0; extra == "formats"
 Provides-Extra: http
 Requires-Dist: httpx[http2]~=0.27; extra == "http"
 Provides-Extra: misc

{omlish-0.0.0.dev14 → omlish-0.0.0.dev15}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev14'
-__revision__ = '6d7ee2004110d9e34b2463374a8ec3961c87e8e2'
+__version__ = '0.0.0.dev15'
+__revision__ = '52225b19abf6c8dc3f10bdccebdc0a7ca9f48ed1'
 
 
 #
@@ -50,10 +50,15 @@ class Project(ProjectBase):
         ],
 
         'formats': [
-            'cloudpickle ~= 3.0',
+            'orjson ~= 3.10',
+            # 'python-rapidjson ~= 1.18',
+            # 'ujson ~= 5.10',
+
             'json5 ~= 0.9',
-            'orjson > 3.10',
+
             'pyyaml ~= 5.0',
+
+            'cloudpickle ~= 3.0',
         ],
 
         'http': [

{omlish-0.0.0.dev14 → omlish-0.0.0.dev15}/omlish/bootstrap.py

@@ -2,6 +2,15 @@
 TODO:
  - more logging options
  - a more powerful interface would be run_fn_with_bootstrap..
+ - ./python -m gprof2dot -f pstats prof.pstats | dot -Tpdf -o prof.pstats.pdf && open prof.pstats.pdf
+
+TODO diag:
+ - yappi
+ - stackscope
+ - https://github.com/pythonspeed/filprofiler
+ - https://pypi.org/project/guppy3/
+ - https://pypi.org/project/memory-profiler/
+ - https://pypi.org/project/Pympler/
 
 TODO new items:
  - pydevd connect-back

{omlish-0.0.0.dev14 → omlish-0.0.0.dev15}/omlish/concurrent.py

@@ -161,20 +161,21 @@ class ImmediateExecutor(cf.Executor):
 
 
 @contextlib.contextmanager
-def new_thread_or_immediate_executor(
+def new_executor(
         max_workers: int | None = None,
+        cls: type[cf.Executor] = cf.ThreadPoolExecutor,
         *,
         immediate_exceptions: bool = False,
-        thread_name_prefix: str = '',
         **kwargs: ta.Any,
 ) -> ta.Generator[cf.Executor, None, None]:
     if max_workers == 0:
         yield ImmediateExecutor(
             immediate_exceptions=immediate_exceptions,
         )
+
     else:
-        with cf.ThreadPoolExecutor(
-                thread_name_prefix=thread_name_prefix,
+        with cls(  # type: ignore
+                max_workers,
                 **kwargs,
         ) as exe:
             yield exe

omlish-0.0.0.dev15/omlish/formats/json.py

@@ -0,0 +1,57 @@
+"""
+TODO:
+ - backend abstr
+ - streaming
+"""
+import functools
+import json as _json
+import typing as ta
+
+from .. import lang
+
+
+if ta.TYPE_CHECKING:
+    import orjson as _orjson
+    import ujson as _ujson
+else:
+    _orjson = lang.proxy_import('orjson')
+    _ujson = lang.proxy_import('ujson')
+
+
+##
+
+
+dump = _json.dump
+dumps = _json.dumps
+
+detect_encoding = _json.detect_encoding
+
+load = _json.load
+loads = _json.loads
+
+
+##
+
+
+PRETTY_INDENT = 2
+
+PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=PRETTY_INDENT,
+)
+
+dump_pretty: ta.Callable[..., bytes] = functools.partial(dump, **PRETTY_KWARGS)  # type: ignore
+dumps_pretty: ta.Callable[..., str] = functools.partial(dumps, **PRETTY_KWARGS)
+
+
+##
+
+
+COMPACT_SEPARATORS = (',', ':')
+
+COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
+    indent=None,
+    separators=COMPACT_SEPARATORS,
+)
+
+dump_compact: ta.Callable[..., bytes] = functools.partial(dump, **COMPACT_KWARGS)  # type: ignore
+dumps_compact: ta.Callable[..., str] = functools.partial(dumps, **COMPACT_KWARGS)

omlish-0.0.0.dev15/omlish/multiprocessing.py

@@ -0,0 +1,173 @@
+import abc
+import dataclasses as dc
+import multiprocessing as mp
+import multiprocessing.popen_spawn_posix
+import os
+import signal
+import sys
+import time
+import typing as ta
+
+from . import check
+from . import libc
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class SpawnExtras:
+    fds: ta.AbstractSet[int] | None = None
+    deathsig: int | None = None
+
+
+class ExtrasSpawnPosixPopen(mp.popen_spawn_posix.Popen):
+    def __init__(self, process_obj: 'ExtrasSpawnProcess', *, extras: SpawnExtras) -> None:
+        self.__extras = extras
+        self.__extra_fds = extras.fds
+        super().__init__(process_obj)
+
+    def _launch(self, process_obj: 'ExtrasSpawnProcess') -> None:
+        if self.__extra_fds:
+            for fd in self.__extra_fds:
+                self.duplicate_for_child(fd)
+            self._extra_fds = None
+        super()._launch(process_obj)  # type: ignore  # noqa
+
+
+class ExtrasSpawnProcess(mp.context.SpawnProcess):
+    def __init__(self, *args: ta.Any, extras: SpawnExtras, **kwargs: ta.Any) -> None:
+        self.__extras = extras
+        super().__init__(*args, **kwargs)
+
+    def _Popen(self, process_obj: 'ExtrasSpawnProcess') -> ExtrasSpawnPosixPopen:  # type: ignore  # noqa
+        return ExtrasSpawnPosixPopen(process_obj, extras=self.__extras)
+
+    def run(self) -> None:
+        if self.__extras.deathsig is not None and sys.platform == 'linux':
+            libc.prctl(libc.PR_SET_PDEATHSIG, self.__extras.deathsig, 0, 0, 0, 0)
+
+        super().run()
+
+
+class ExtrasSpawnContext(mp.context.SpawnContext):
+    def __init__(self, extras: SpawnExtras = SpawnExtras()) -> None:
+        super().__init__()
+        self.__extras = extras
+
+    def Process(self, *args: ta.Any, **kwargs: ta.Any):  # type: ignore  # noqa
+        return ExtrasSpawnProcess(*args, extras=self.__extras, **kwargs)
+
+
+##
+
+
+class Deathpact(abc.ABC):
+    @abc.abstractmethod
+    def poll(self) -> None:
+        raise NotImplementedError
+
+
+#
+
+
+class BaseDeathpact(Deathpact, abc.ABC):
+    def __init__(
+            self,
+            *,
+            interval_s: float = .5,
+            signal: int | None = signal.SIGTERM,
+            output: ta.Literal['stdout', 'stderr'] | None = 'stderr',
+            on_die: ta.Callable[[], None] | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._interval_s = interval_s
+        self._signal = signal
+        self._output = output
+        self._on_die = on_die
+
+        self._last_check_t: float | None = None
+
+    def _print(self, msg: str) -> None:
+        match self._output:
+            case 'stdout':
+                f = sys.stdout
+            case 'stderr':
+                f = sys.stderr
+            case _:
+                return
+        print(f'{self} pid={os.getpid()}: {msg}', file=f)
+
+    def die(self) -> None:
+        self._print('Triggered! Process terminating!')
+
+        if self._on_die is not None:
+            self._on_die()
+
+        if self._signal is not None:
+            os.kill(os.getpid(), self._signal)
+
+        sys.exit(1)
+
+    @abc.abstractmethod
+    def should_die(self) -> bool:
+        raise NotImplementedError
+
+    def maybe_die(self) -> None:
+        if self.should_die():
+            self.die()
+
+    def poll(self) -> None:
+        if self._last_check_t is None or (time.monotonic() - self._last_check_t) >= self._interval_s:
+            self.maybe_die()
+            self._last_check_t = time.monotonic()
+
+
+#
+
+
+class PipeDeathpact(BaseDeathpact):
+    def __init__(self, **kwargs: ta.Any) -> None:
+        super().__init__(**kwargs)
+
+        self._rfd: int | None = None
+        self._wfd: int | None = None
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}(rfd={self._rfd}, wfd={self._wfd})'
+
+    @property
+    def fd(self) -> int:
+        return check.not_none(self._rfd)
+
+    def __enter__(self) -> ta.Self:
+        check.none(self._rfd)
+        check.none(self._wfd)
+
+        self._rfd, self._wfd = os.pipe()
+
+        os.set_inheritable(self._rfd, True)
+        os.set_blocking(self._rfd, False)
+
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._rfd is not None:
+            os.close(check.not_none(self._wfd))
+            self._wfd = None
+
+            os.close(self._rfd)
+            self._rfd = None
+
+    def should_die(self) -> bool:
+        try:
+            buf = os.read(check.not_none(self._rfd), 1)
+        except BlockingIOError:
+            return False
+
+        if buf:
+            self._print(f'Read data from pipe! This should not happen! Process state corrupt!')
+            self.die()
+
+        return True

{omlish-0.0.0.dev14 → omlish-0.0.0.dev15/omlish.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev14
+Version: 0.0.0.dev15
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause
@@ -22,10 +22,10 @@ Requires-Dist: lz4~=4.0; extra == "all"
 Requires-Dist: python-snappy~=0.7; python_version < "3.13" and extra == "all"
 Requires-Dist: zstd~=1.5; extra == "all"
 Requires-Dist: psutil~=6.0; extra == "all"
-Requires-Dist: cloudpickle~=3.0; extra == "all"
+Requires-Dist: orjson~=3.10; extra == "all"
 Requires-Dist: json5~=0.9; extra == "all"
-Requires-Dist: orjson>3.10; extra == "all"
 Requires-Dist: pyyaml~=5.0; extra == "all"
+Requires-Dist: cloudpickle~=3.0; extra == "all"
 Requires-Dist: httpx[http2]~=0.27; extra == "all"
 Requires-Dist: jinja2~=3.1; extra == "all"
 Requires-Dist: wrapt~=1.14; extra == "all"
@@ -53,10 +53,10 @@ Requires-Dist: zstd~=1.5; extra == "compression"
 Provides-Extra: diag
 Requires-Dist: psutil~=6.0; extra == "diag"
 Provides-Extra: formats
-Requires-Dist: cloudpickle~=3.0; extra == "formats"
+Requires-Dist: orjson~=3.10; extra == "formats"
 Requires-Dist: json5~=0.9; extra == "formats"
-Requires-Dist: orjson>3.10; extra == "formats"
 Requires-Dist: pyyaml~=5.0; extra == "formats"
+Requires-Dist: cloudpickle~=3.0; extra == "formats"
 Provides-Extra: http
 Requires-Dist: httpx[http2]~=0.27; extra == "http"
 Provides-Extra: misc

{omlish-0.0.0.dev14 → omlish-0.0.0.dev15}/omlish.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ omlish/iterators.py
 omlish/libc.py
 omlish/matchfns.py
 omlish/math.py
+omlish/multiprocessing.py
 omlish/os.py
 omlish/runmodule.py
 omlish/stats.py

{omlish-0.0.0.dev14 → omlish-0.0.0.dev15}/omlish.egg-info/requires.txt

@@ -6,10 +6,10 @@ trio~=0.26
 lz4~=4.0
 zstd~=1.5
 psutil~=6.0
-cloudpickle~=3.0
+orjson~=3.10
 json5~=0.9
-orjson>3.10
 pyyaml~=5.0
+cloudpickle~=3.0
 httpx[http2]~=0.27
 jinja2~=3.1
 wrapt~=1.14
@@ -52,10 +52,10 @@ python-snappy~=0.7
 psutil~=6.0
 
 [formats]
-cloudpickle~=3.0
+orjson~=3.10
 json5~=0.9
-orjson>3.10
 pyyaml~=5.0
+cloudpickle~=3.0
 
 [http]
 httpx[http2]~=0.27

{omlish-0.0.0.dev14 → omlish-0.0.0.dev15}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 urls = {source = 'https://github.com/wrmsr/omlish'}
 license = {text = 'BSD-3-Clause'}
 requires-python = '~=3.12'
-version = '0.0.0.dev14'
+version = '0.0.0.dev15'
 classifiers = [
     'License :: OSI Approved :: BSD License',
     'Development Status :: 2 - Pre-Alpha',
@@ -33,10 +33,10 @@ all = [
     'python-snappy ~= 0.7; python_version < "3.13"',
     'zstd ~= 1.5',
     'psutil ~= 6.0',
-    'cloudpickle ~= 3.0',
+    'orjson ~= 3.10',
     'json5 ~= 0.9',
-    'orjson > 3.10',
     'pyyaml ~= 5.0',
+    'cloudpickle ~= 3.0',
     'httpx[http2] ~= 0.27',
     'jinja2 ~= 3.1',
     'wrapt ~= 1.14',
@@ -68,10 +68,10 @@ diag = [
     'psutil ~= 6.0',
 ]
 formats = [
-    'cloudpickle ~= 3.0',
+    'orjson ~= 3.10',
     'json5 ~= 0.9',
-    'orjson > 3.10',
     'pyyaml ~= 5.0',
+    'cloudpickle ~= 3.0',
 ]
 http = [
     'httpx[http2] ~= 0.27',