omdev 0.0.0.dev215__tar.gz → 0.0.0.dev217__tar.gz

{omdev-0.0.0.dev215/omdev.egg-info → omdev-0.0.0.dev217}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev215
+Version: 0.0.0.dev217
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev215
+Requires-Dist: omlish==0.0.0.dev217
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev215 → omdev-0.0.0.dev217}/omdev/.manifests.json

@@ -195,7 +195,7 @@
     "module": ".scripts.exectime",
     "attr": "_CLI_MODULE",
     "file": "omdev/scripts/exectime.py",
-    "line": 7,
+    "line": 66,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "py/exectime",

{omdev-0.0.0.dev215 → omdev-0.0.0.dev217}/omdev/precheck/lite.py

@@ -25,13 +25,46 @@ log = logging.getLogger(__name__)
 class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
     @dc.dataclass(frozen=True)
     class Config(Precheck.Config):
-        pass
+        python: str = '.venvs/8/bin/python'
+        concurrency: int = 4
 
     def __init__(self, context: PrecheckContext, config: Config = Config()) -> None:
         super().__init__(context, config)
 
     #
 
+    @dc.dataclass(frozen=True)
+    class _Target:
+        path: str
+        kind: ta.Literal['script', 'module']
+
+    async def _collect_targets(self) -> list[_Target]:
+        lst = []
+
+        for fp in magic.find_magic_files(
+                magic.PY_MAGIC_STYLE,
+                self._context.src_roots,
+                keys=['@omlish-lite'],
+        ):
+            with open(fp) as f:  # noqa  # FIXME
+                src = f.read()
+
+            is_script = '# @omlish-script' in src.splitlines()
+
+            if is_script:
+                lst.append(self._Target(fp, 'script'))
+
+            elif fp.endswith('__init__.py'):
+                for g in glob.glob(os.path.join(os.path.dirname(fp), '**/*.py'), recursive=True):
+                    lst.append(self._Target(g, 'module'))
+
+            else:
+                lst.append(self._Target(fp, 'module'))
+
+        return lst
+
+    #
+
     @staticmethod
     def _load_file_module(fp: str) -> None:
         import os.path  # noqa
@@ -71,7 +104,7 @@ class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
 
         proc = await asyncio.create_subprocess_exec(
             *subprocess_maybe_shell_wrap_exec(
-                '.venvs/8/bin/python',
+                self._config.python,
                 '-c',
                 self._load_file_module_payload(),
                 fp,
@@ -85,7 +118,7 @@ class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
 
         return vs
 
-    async def _run_one_module(self, fp: str) -> list[Precheck.Violation]:
+    async def _run_module(self, fp: str) -> list[Precheck.Violation]:
         vs: list[Precheck.Violation] = []
 
         mod = fp.rpartition('.')[0].replace(os.sep, '.')
@@ -94,7 +127,7 @@ class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
 
         proc = await asyncio.create_subprocess_exec(
             *subprocess_maybe_shell_wrap_exec(
-                '.venvs/8/bin/python',
+                self._config.python,
                 '-c',
                 f'import {mod}',
             ),
@@ -107,34 +140,29 @@ class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
 
         return vs
 
-    async def _run_module(self, fp: str) -> list[Precheck.Violation]:
-        vs: list[Precheck.Violation] = []
+    #
 
-        if fp.endswith('__init__.py'):
-            pfps = glob.glob(os.path.join(os.path.dirname(fp), '**/*.py'), recursive=True)
-        else:
-            pfps = [fp]
+    async def _run_one(self, tgt: _Target) -> list[Precheck.Violation]:
+        if tgt.kind == 'script':
+            return await self._run_script(tgt.path)
 
-        for pfp in pfps:
-            vs.extend(await self._run_one_module(pfp))
+        elif tgt.kind == 'module':
+            return await self._run_module(tgt.path)
 
-        return vs
+        else:
+            raise RuntimeError(f'Unknown target kind: {tgt.kind}')
 
     async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
-        for fp in magic.find_magic_files(
-                magic.PY_MAGIC_STYLE,
-                self._context.src_roots,
-                keys=['@omlish-lite'],
-        ):
-            with open(fp) as f:  # noqa  # FIXME
-                src = f.read()
+        tgts = await self._collect_targets()
 
-            is_script = '# @omlish-script' in src.splitlines()
+        sem = asyncio.Semaphore(self._config.concurrency)
 
-            if is_script:
-                for v in await self._run_script(fp):
-                    yield v
+        async def run(tgt):
+            async with sem:
+                return await self._run_one(tgt)
 
-            else:
-                for v in await self._run_module(fp):
-                    yield v
+        tasks = [asyncio.create_task(run(tgt)) for tgt in tgts]
+
+        for coro in asyncio.as_completed(tasks):
+            for v in await coro:
+                yield v

{omdev-0.0.0.dev215 → omdev-0.0.0.dev217}/omdev/precheck/main.py

@@ -60,6 +60,7 @@ def _check_cmd(args) -> None:
         vs: list[Precheck.Violation] = []
 
         for pc in pcs:
+            log.info('Running precheck: %s', type(pc).__name__)
             async for v in pc.run():
                 vs.append(v)
                 print('*** VIOLATION ***')

{omdev-0.0.0.dev215 → omdev-0.0.0.dev217}/omdev/scripts/ci.py

@@ -167,7 +167,7 @@ def asyncio_once(fn: CallableT) -> CallableT:
     return ta.cast(CallableT, inner)
 
 
-def drain_tasks(loop=None):
+def drain_asyncio_tasks(loop=None):
     if loop is None:
         loop = asyncio.get_running_loop()
 
@@ -182,7 +182,7 @@ def draining_asyncio_tasks() -> ta.Iterator[None]:
         yield
     finally:
         if loop is not None:
-            drain_tasks(loop)  # noqa
+            drain_asyncio_tasks(loop)  # noqa
 
 
 async def asyncio_wait_concurrent(
@@ -2563,7 +2563,7 @@ TODO:
 
 STANDARD_LOG_FORMAT_PARTS = [
     ('asctime', '%(asctime)-15s'),
-    ('process', 'pid=%(process)-6s'),
+    ('process', 'pid=%(process)s'),
     ('thread', 'tid=%(thread)x'),
     ('levelname', '%(levelname)s'),
     ('name', '%(name)s'),
@@ -2676,6 +2676,12 @@ def configure_standard_logging(
 ##
 
 
+# Valid channel type kwarg values:
+#  - A special flag negative int
+#  - A positive fd int
+#  - A file-like object
+#  - None
+
 SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
     'pipe': subprocess.PIPE,
     'stdout': subprocess.STDOUT,
@@ -2721,6 +2727,25 @@ def subprocess_close(
 ##
 
 
+class VerboseCalledProcessError(subprocess.CalledProcessError):
+    @classmethod
+    def from_std(cls, e: subprocess.CalledProcessError) -> 'VerboseCalledProcessError':
+        return cls(
+            e.returncode,
+            e.cmd,
+            output=e.output,
+            stderr=e.stderr,
+        )
+
+    def __str__(self) -> str:
+        msg = super().__str__()
+        if self.output is not None:
+            msg += f' Output: {self.output!r}'
+        if self.stderr is not None:
+            msg += f' Stderr: {self.stderr!r}'
+        return msg
+
+
 class BaseSubprocesses(abc.ABC):  # noqa
     DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
 
@@ -2754,16 +2779,31 @@ class BaseSubprocesses(abc.ABC):  # noqa
             if extra_env:
                 self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
 
+        #
+
         if extra_env:
             env = {**(env if env is not None else os.environ), **extra_env}
 
+        #
+
         if quiet and 'stderr' not in kwargs:
             if self._log and not self._log.isEnabledFor(logging.DEBUG):
                 kwargs['stderr'] = subprocess.DEVNULL
 
+        for chk in ('stdout', 'stderr'):
+            try:
+                chv = kwargs[chk]
+            except KeyError:
+                continue
+            kwargs[chk] = SUBPROCESS_CHANNEL_OPTION_VALUES.get(chv, chv)
+
+        #
+
         if not shell:
             cmd = subprocess_maybe_shell_wrap_exec(*cmd)
 
+        #
+
         return cmd, dict(
             env=env,
             shell=shell,
@@ -2771,35 +2811,57 @@ class BaseSubprocesses(abc.ABC):  # noqa
         )
 
     @contextlib.contextmanager
-    def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
+    def wrap_call(
+            self,
+            *cmd: ta.Any,
+            raise_verbose: bool = False,
+            **kwargs: ta.Any,
+    ) -> ta.Iterator[None]:
         start_time = time.time()
         try:
             if self._log:
                 self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
+
             yield
 
         except Exception as exc:  # noqa
             if self._log:
                 self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
+
+            if (
+                    raise_verbose and
+                    isinstance(exc, subprocess.CalledProcessError) and
+                    not isinstance(exc, VerboseCalledProcessError) and
+                    (exc.output is not None or exc.stderr is not None)
+            ):
+                raise VerboseCalledProcessError.from_std(exc) from exc
+
             raise
 
         finally:
             end_time = time.time()
             elapsed_s = end_time - start_time
+
             if self._log:
-                self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
+                self._log.debug('Subprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
 
     @contextlib.contextmanager
     def prepare_and_wrap(
             self,
             *cmd: ta.Any,
+            raise_verbose: bool = False,
             **kwargs: ta.Any,
     ) -> ta.Iterator[ta.Tuple[
         ta.Tuple[ta.Any, ...],
         ta.Dict[str, ta.Any],
     ]]:
         cmd, kwargs = self.prepare_args(*cmd, **kwargs)
-        with self.wrap_call(*cmd, **kwargs):
+
+        with self.wrap_call(
+                *cmd,
+                raise_verbose=raise_verbose,
+                **kwargs,
+        ):
             yield cmd, kwargs
 
     #

omdev-0.0.0.dev217/omdev/scripts/exectime.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# @omlish-script
+
+
+##
+
+
+def _run_one(src, pre=None):
+    import time
+
+    if pre:
+        exec(pre)
+
+    co = compile(src, '<string>', 'exec')
+    start = time.time_ns()
+    exec(co)
+    end = time.time_ns()
+
+    return end - start
+
+
+##
+
+
+def _run(n, src, pre=None):
+    if n is None:
+        return _run_one(src, pre=pre)
+
+    #
+
+    import inspect
+
+    cmd = '\n'.join([
+        inspect.getsource(_run_one),
+        f'print(_run_one({src!r}, pre={pre!r}))',
+    ])
+
+    #
+
+    import subprocess
+    import sys
+
+    ts = []
+    for _ in range(n):
+        out = subprocess.check_output([sys.executable, '-c', cmd]).decode()
+        t = int(out.strip())
+        ts.append(t)
+
+    #
+
+    import statistics
+
+    o = {
+        # 'times': ts,
+        'mean': statistics.mean(ts),
+        'median': statistics.median(ts),
+        'quantiles': statistics.quantiles(ts),
+    }
+
+    return o
+
+
+##
+
+
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': 'py/exectime',
+    'mod_name': __name__,
+}}
+
+
+def _main():
+    import sys
+
+    args = sys.argv[1:]
+
+    n = None
+    if args:
+        try:
+            n = int(args[0])
+        except ValueError:
+            pass
+        else:
+            args.pop(0)
+
+    if len(args) > 1:
+        pre = args.pop(0)
+    else:
+        pre = None
+
+    if len(args) != 1:
+        raise Exception('Invalid arguments')
+    [src] = args
+
+    #
+
+    o = _run(n, src, pre=pre)
+
+    import json
+
+    print(json.dumps(o, indent=2))
+
+
+if __name__ == '__main__':
+    _main()

{omdev-0.0.0.dev215 → omdev-0.0.0.dev217}/omdev/scripts/interp.py

@@ -3500,7 +3500,7 @@ TODO:
 
 STANDARD_LOG_FORMAT_PARTS = [
     ('asctime', '%(asctime)-15s'),
-    ('process', 'pid=%(process)-6s'),
+    ('process', 'pid=%(process)s'),
     ('thread', 'tid=%(thread)x'),
     ('levelname', '%(levelname)s'),
     ('name', '%(name)s'),
@@ -3613,6 +3613,12 @@ def configure_standard_logging(
 ##
 
 
+# Valid channel type kwarg values:
+#  - A special flag negative int
+#  - A positive fd int
+#  - A file-like object
+#  - None
+
 SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
     'pipe': subprocess.PIPE,
     'stdout': subprocess.STDOUT,
@@ -3658,6 +3664,25 @@ def subprocess_close(
 ##
 
 
+class VerboseCalledProcessError(subprocess.CalledProcessError):
+    @classmethod
+    def from_std(cls, e: subprocess.CalledProcessError) -> 'VerboseCalledProcessError':
+        return cls(
+            e.returncode,
+            e.cmd,
+            output=e.output,
+            stderr=e.stderr,
+        )
+
+    def __str__(self) -> str:
+        msg = super().__str__()
+        if self.output is not None:
+            msg += f' Output: {self.output!r}'
+        if self.stderr is not None:
+            msg += f' Stderr: {self.stderr!r}'
+        return msg
+
+
 class BaseSubprocesses(abc.ABC):  # noqa
     DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
 
@@ -3691,16 +3716,31 @@ class BaseSubprocesses(abc.ABC):  # noqa
             if extra_env:
                 self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
 
+        #
+
         if extra_env:
             env = {**(env if env is not None else os.environ), **extra_env}
 
+        #
+
         if quiet and 'stderr' not in kwargs:
             if self._log and not self._log.isEnabledFor(logging.DEBUG):
                 kwargs['stderr'] = subprocess.DEVNULL
 
+        for chk in ('stdout', 'stderr'):
+            try:
+                chv = kwargs[chk]
+            except KeyError:
+                continue
+            kwargs[chk] = SUBPROCESS_CHANNEL_OPTION_VALUES.get(chv, chv)
+
+        #
+
         if not shell:
             cmd = subprocess_maybe_shell_wrap_exec(*cmd)
 
+        #
+
         return cmd, dict(
             env=env,
             shell=shell,
@@ -3708,35 +3748,57 @@ class BaseSubprocesses(abc.ABC):  # noqa
         )
 
     @contextlib.contextmanager
-    def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
+    def wrap_call(
+            self,
+            *cmd: ta.Any,
+            raise_verbose: bool = False,
+            **kwargs: ta.Any,
+    ) -> ta.Iterator[None]:
         start_time = time.time()
         try:
             if self._log:
                 self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
+
             yield
 
         except Exception as exc:  # noqa
             if self._log:
                 self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
+
+            if (
+                    raise_verbose and
+                    isinstance(exc, subprocess.CalledProcessError) and
+                    not isinstance(exc, VerboseCalledProcessError) and
+                    (exc.output is not None or exc.stderr is not None)
+            ):
+                raise VerboseCalledProcessError.from_std(exc) from exc
+
             raise
 
         finally:
             end_time = time.time()
             elapsed_s = end_time - start_time
+
             if self._log:
-                self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
+                self._log.debug('Subprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
 
     @contextlib.contextmanager
     def prepare_and_wrap(
             self,
             *cmd: ta.Any,
+            raise_verbose: bool = False,
             **kwargs: ta.Any,
     ) -> ta.Iterator[ta.Tuple[
         ta.Tuple[ta.Any, ...],
         ta.Dict[str, ta.Any],
     ]]:
         cmd, kwargs = self.prepare_args(*cmd, **kwargs)
-        with self.wrap_call(*cmd, **kwargs):
+
+        with self.wrap_call(
+                *cmd,
+                raise_verbose=raise_verbose,
+                **kwargs,
+        ):
             yield cmd, kwargs
 
     #