omdev 0.0.0.dev97__tar.gz → 0.0.0.dev98__tar.gz

{omdev-0.0.0.dev97/omdev.egg-info → omdev-0.0.0.dev98}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev97
+Version: 0.0.0.dev98
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,13 +12,14 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev97
+Requires-Dist: omlish==0.0.0.dev98
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"
-Requires-Dist: cffi~=1.17; extra == "all"
 Requires-Dist: pcpp~=1.30; extra == "all"
+Requires-Dist: cffi~=1.17; extra == "all"
 Requires-Dist: docutils~=0.21; extra == "all"
+Requires-Dist: markdown~=3.7; extra == "all"
 Requires-Dist: mypy~=1.11; extra == "all"
 Requires-Dist: gprof2dot~=2024.6; extra == "all"
 Requires-Dist: prompt-toolkit~=3.0; extra == "all"
@@ -28,10 +29,11 @@ Provides-Extra: black
 Requires-Dist: black~=24.10; extra == "black"
 Provides-Extra: c
 Requires-Dist: pycparser~=2.22; extra == "c"
-Requires-Dist: cffi~=1.17; extra == "c"
 Requires-Dist: pcpp~=1.30; extra == "c"
-Provides-Extra: docutils
-Requires-Dist: docutils~=0.21; extra == "docutils"
+Requires-Dist: cffi~=1.17; extra == "c"
+Provides-Extra: doc
+Requires-Dist: docutils~=0.21; extra == "doc"
+Requires-Dist: markdown~=3.7; extra == "doc"
 Provides-Extra: mypy
 Requires-Dist: mypy~=1.11; extra == "mypy"
 Provides-Extra: prof

{omdev-0.0.0.dev97 → omdev-0.0.0.dev98}/omdev/.manifests.json

@@ -203,6 +203,18 @@
       }
     }
   },
+  {
+    "module": ".tools.doctools",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/doctools.py",
+    "line": 103,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "doc",
+        "mod_name": "omdev.tools.doctools"
+      }
+    }
+  },
   {
     "module": ".tools.gittools",
     "attr": "_CLI_MODULE",
@@ -254,6 +266,18 @@
       }
     }
   },
+  {
+    "module": ".tools.pawk.pawk",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/pawk/pawk.py",
+    "line": 390,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "pawk",
+        "mod_name": "omdev.tools.pawk.pawk"
+      }
+    }
+  },
   {
     "module": ".tools.piptools",
     "attr": "_CLI_MODULE",
@@ -278,18 +302,6 @@
       }
     }
   },
-  {
-    "module": ".tools.rsttool",
-    "attr": "_CLI_MODULE",
-    "file": "omdev/tools/rsttool.py",
-    "line": 55,
-    "value": {
-      "$.cli.types.CliModule": {
-        "cmd_name": "rst",
-        "mod_name": "omdev.tools.rsttool"
-      }
-    }
-  },
   {
     "module": ".tools.sqlrepl",
     "attr": "_CLI_MODULE",

{omdev-0.0.0.dev97 → omdev-0.0.0.dev98}/omdev/__about__.py

@@ -18,12 +18,15 @@ class Project(ProjectBase):
 
         'c': [
             'pycparser ~= 2.22',
-            'cffi ~= 1.17',
             'pcpp ~= 1.30',
+
+            'cffi ~= 1.17',
         ],
 
-        'docutils': [
+        'doc': [
             'docutils ~= 0.21',
+
+            'markdown ~= 3.7',
         ],
 
         'mypy': [

omdev-0.0.0.dev98/omdev/tools/doctools.py

@@ -0,0 +1,108 @@
+"""
+TODO:
+ - omdev/rst.py *and* rsttool.py, when we want extracted helpers
+"""
+import contextlib
+import io
+import os.path
+import subprocess
+import sys
+import tempfile
+import typing as ta
+
+from omlish import argparse as ap
+from omlish import check
+from omlish import lang
+
+from ..cli import CliModule
+
+
+if ta.TYPE_CHECKING:
+    import docutils.core
+    import markdown
+else:
+    docutils = lang.proxy_import('docutils', extras=['core'])
+    markdown = lang.proxy_import('markdown')
+
+
+def rst2html(rst, report_level=None):
+    kwargs = {
+        'writer_name': 'html',
+        'settings_overrides': {
+            '_disable_config': True,
+            'report_level': int(report_level) if report_level else 0,
+        },
+    }
+
+    target = io.StringIO()
+    with contextlib.redirect_stderr(target):
+        parts = docutils.core.publish_parts(rst, **kwargs)  # type: ignore
+        html = parts['html_body']
+        warning = target.getvalue().strip()
+        return html, warning
+
+
+def open_html(src: str, name: str) -> None:
+    check.non_empty_str(name)
+    out_dir = tempfile.mkdtemp()
+    out_file = os.path.join(out_dir, name + '.html')
+    with open(out_file, 'w') as f:
+        f.write(src)
+    subprocess.check_call(['open', out_file])
+
+
+class Cli(ap.Cli):
+    def _read_file(self, input_file: str | None) -> tuple[str, str]:
+        if input_file is not None:
+            with open(input_file) as f:
+                src = f.read()
+            name = os.path.basename(input_file)
+
+        else:
+            src = sys.stdin.read()
+            name = 'stdin'
+
+        return src, name
+
+    @ap.command(
+        ap.arg('input-file', nargs='?'),
+        ap.arg('--report-level', type=int),
+        ap.arg('-O', '--open', action='store_true'),
+    )
+    def rst(self) -> None:
+        src, name = self._read_file(self.args.input_file)
+
+        html, warning = rst2html(
+            src,
+            report_level=self.args.report_level,
+        )
+
+        if warning:
+            sys.stderr.write(warning)
+
+        if self.args.open:
+            open_html(html, name)
+        else:
+            print(html)
+
+    @ap.command(
+        ap.arg('input-file', nargs='?'),
+        ap.arg('-O', '--open', action='store_true'),
+    )
+    def md(self):
+        src, name = self._read_file(self.args.input_file)
+
+        html = markdown.markdown(src)
+
+        if self.args.open:
+            open_html(html, name)
+        else:
+            print(html)
+
+
+# @omlish-manifest
+_CLI_MODULE = CliModule('doc', __name__)
+
+
+if __name__ == '__main__':
+    Cli().call_and_exit()

omdev-0.0.0.dev98/omdev/tools/pawk/__main__.py

@@ -0,0 +1,4 @@
+if __name__ == '__main__':
+    from .pawk import main
+
+    main()

omdev-0.0.0.dev98/omdev/tools/pawk/pawk.py

@@ -0,0 +1,395 @@
+#!/usr/bin/env python
+# Copyright (C) 2018 Alec Thomas
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# https://github.com/alecthomas/pawk/blob/d60f78399e8a01857ebd73415a00e7eb424043ab/pawk.py
+"""cat input | pawk [<options>] <expr>
+
+A Python line-processor (like awk).
+
+See https://github.com/alecthomas/pawk for details. Based on
+http://code.activestate.com/recipes/437932/.
+"""
+import argparse
+import ast
+import codecs
+import contextlib
+import inspect
+import itertools
+import os
+import re
+import sys
+import types
+import typing as ta
+
+from omlish import check
+
+from ...cli import CliModule
+
+
+RESULT_VAR_NAME = '__result'
+
+STRING_ESCAPE = 'unicode_escape'
+
+
+# Store the last expression, if present, into variable var_name.
+def save_last_expression(
+        tree: ast.Module,
+        var_name: str = RESULT_VAR_NAME,
+) -> ast.Module:
+    body = tree.body
+
+    node: ast.AST | None = body[-1] if len(body) else None
+
+    body.insert(
+        0,
+        ast.Assign(
+            targets=[
+                ast.Name(
+                    id=var_name,
+                    ctx=ast.Store(),
+                ),
+            ],
+            value=ast.Constant(None),
+        ),
+    )
+
+    if node is not None and isinstance(node, ast.Expr):
+        body[-1] = ast.copy_location(
+            ast.Assign(
+                targets=[
+                    ast.Name(
+                        id=var_name,
+                        ctx=ast.Store(),
+                    ),
+                ],
+                value=node.value,
+            ),
+            node,  # noqa
+        )
+
+    return ast.fix_missing_locations(tree)
+
+
+def compile_command(text: str) -> types.CodeType:
+    mod = check.isinstance(compile(text, 'EXPR', 'exec', flags=ast.PyCF_ONLY_AST), ast.Module)
+    tree = save_last_expression(mod)
+    return compile(tree, 'EXPR', 'exec')
+
+
+def eval_in_context(
+        codeobj: types.CodeType,
+        context: 'Context',
+        var_name: str = RESULT_VAR_NAME,
+) -> ta.Any:
+    exec(codeobj, globals(), context)
+    return context.pop(var_name, None)
+
+
+class Action:
+    """Represents a single action to be applied to each line."""
+
+    def __init__(
+            self,
+            pattern: str | None = None,
+            cmd: str = 'l',
+            have_end_statement: bool = False,
+            negate: bool = False,
+            strict: bool = False,
+    ) -> None:
+        super().__init__()
+
+        self.delim = None
+        self.odelim = ' '
+        self.negate = negate
+        self.pattern = None if pattern is None else re.compile(pattern)
+        self.cmd = cmd
+        self.strict = strict
+        self._compile(have_end_statement)
+
+    @classmethod
+    def from_options(cls, options: ta.Any, arg: str) -> 'Action':
+        negate, pattern, cmd = Action._parse_command(arg)
+        return cls(
+            pattern=pattern,
+            cmd=cmd,
+            have_end_statement=(options.end is not None),
+            negate=negate,
+            strict=options.strict,
+        )
+
+    def _compile(self, have_end_statement: bool) -> None:
+        if not self.cmd:
+            if have_end_statement:
+                self.cmd = 't += line'
+            else:
+                self.cmd = 'l'
+        self._codeobj = compile_command(self.cmd)
+
+    def apply(self, context: 'Context', line: str) -> ta.Any:
+        """
+        Apply action to line.
+
+        :return: Line text or None.
+        """
+
+        match = self._match(line)
+        if match is None:
+            return None
+        context['m'] = match
+        try:
+            return eval_in_context(self._codeobj, context)
+        except Exception:  # noqa
+            if not self.strict:
+                return None
+            raise
+
+    def _match(self, line: str) -> ta.Any:
+        if self.pattern is None:
+            return self.negate
+        match = self.pattern.search(line)
+        if match is not None:
+            return None if self.negate else match.groups()
+        elif self.negate:
+            return ()
+        else:
+            return None
+
+    @staticmethod
+    def _parse_command(arg: str) -> tuple[bool, str | None, str]:
+        match = re.match(r'(?ms)(?:(!)?/((?:\\.|[^/])+)/)?(.*)', arg)
+        negate, pattern, cmd = check.not_none(match).groups()
+        cmd = cmd.strip()
+        negate = bool(negate)
+        return negate, pattern, cmd
+
+
+class Context(dict):
+    def apply(
+            self,
+            numz: int,
+            line: str,
+            headers: ta.Sequence[str] | None = None,
+    ) -> None:
+        l = line.rstrip()
+        f = l.split(self.delim)
+        self.update(line=line, l=l, n=numz + 1, f=f, nf=len(f))
+        if headers:
+            self.update(itertools.zip_longest(headers, f))
+
+    delim: str | None
+    odelim: str
+    line_separator: str
+
+    @classmethod
+    def from_options(
+            cls,
+            options: ta.Any,
+            modules: ta.Sequence[str],
+    ) -> 'Context':
+        self = cls()
+        self['t'] = ''
+        self['m'] = ()
+        if options.imports:
+            for imp in options.imports.split(','):
+                mod = __import__(imp.strip(), fromlist=['.'])
+                self.update((k, v) for k, v in inspect.getmembers(mod) if k[0] != '_')
+
+        self.delim = codecs.decode(options.delim, STRING_ESCAPE) if options.delim else None
+        self.odelim = codecs.decode(options.delim_out, STRING_ESCAPE)
+        self.line_separator = codecs.decode(options.line_separator, STRING_ESCAPE)
+
+        for m in modules:
+            try:
+                key = m.split('.')[0]
+                self[key] = __import__(m)
+            except Exception:  # noqa
+                pass
+        return self
+
+
+def process(
+        context: Context,
+        input: ta.TextIO,  # noqa
+        output: ta.TextIO,
+        begin_statement: str | None,
+        actions: ta.Sequence[Action],
+        end_statement: str | None,
+        strict: bool,
+        header: bool,
+):
+    """Process a stream."""
+
+    # Override "print"
+    old_stdout = sys.stdout
+    sys.stdout = output
+    write = output.write
+
+    def write_result(result, when_true=None):
+        if result is True:
+            result = when_true
+        elif isinstance(result, (list, tuple)):
+            result = context.odelim.join(map(str, result))
+
+        if result is not None and result is not False:
+            result = str(result)
+            if not result.endswith(context.line_separator):
+                result = result.rstrip('\n') + context.line_separator
+            write(result)
+
+    try:
+        headers = None
+        if header:
+            line = input.readline()
+            context.apply(-1, line)
+            headers = context['f']
+
+        if begin_statement:
+            write_result(eval_in_context(compile_command(begin_statement), context))
+
+        for numz, line in enumerate(input):
+            context.apply(numz, line, headers=headers)
+            for action in actions:
+                write_result(action.apply(context, line), when_true=line)
+
+        if end_statement:
+            write_result(eval_in_context(compile_command(end_statement), context))
+
+    finally:
+        sys.stdout = old_stdout
+
+
+def parse_commandline(argv: ta.Sequence[str]) -> tuple[ta.Any, ta.Sequence[str]]:
+    parser = argparse.ArgumentParser()
+    # parser.set_usage(__doc__.strip())
+    parser.add_argument(
+        '-I',
+        '--in_place',
+        dest='in_place',
+        help='modify given input file in-place',
+        metavar='<filename>',
+    )
+    parser.add_argument(
+        '-i',
+        '--import',
+        dest='imports',
+        help='comma-separated list of modules to "from x import *" from',
+        metavar='<modules>',
+    )
+    parser.add_argument(
+        '-F',
+        dest='delim',
+        help='input delimiter',
+        metavar='<delim>',
+        default=None,
+    )
+    parser.add_argument(
+        '-O',
+        dest='delim_out',
+        help='output delimiter',
+        metavar='<delim>',
+        default=' ',
+    )
+    parser.add_argument(
+        '-L',
+        dest='line_separator',
+        help='output line separator',
+        metavar='<delim>',
+        default='\n',
+    )
+    parser.add_argument(
+        '-B',
+        '--begin',
+        help='begin statement',
+        metavar='<statement>',
+    )
+    parser.add_argument(
+        '-E',
+        '--end',
+        help='end statement',
+        metavar='<statement>',
+    )
+    parser.add_argument(
+        '-s',
+        '--statement',
+        action='store_true',
+        help='DEPRECATED. retained for backward compatibility',
+    )
+    parser.add_argument(
+        '-H',
+        '--header',
+        action='store_true',
+        help='use first row as field variable names in subsequent rows',
+    )
+    parser.add_argument(
+        '--strict',
+        action='store_true',
+        help='abort on exceptions',
+    )
+    return parser.parse_known_args(argv[1:])
+
+
+# For integration tests.
+def run(
+        argv: ta.Sequence[str],
+        input: ta.TextIO,  # noqa
+        output: ta.TextIO,
+) -> None:
+    options, args = parse_commandline(argv)
+
+    with contextlib.ExitStack() as es:
+        if options.in_place:
+            os.rename(options.in_place, options.in_place + '~')
+            input = es.enter_context(open(options.in_place + '~'))  # noqa
+            output = es.enter_context(open(options.in_place, 'w'))
+
+        # Auto-import. This is not smart.
+        all_text = ' '.join([(options.begin or ''), ' '.join(args), (options.end or '')])
+        modules = re.findall(r'([\w.]+)+(?=\.\w+)\b', all_text)
+
+        context = Context.from_options(options, modules)
+        actions = [Action.from_options(options, arg) for arg in args]
+        if not actions:
+            actions = [Action.from_options(options, '')]
+
+        process(
+            context,
+            input,
+            output,
+            options.begin,
+            actions,
+            options.end,
+            options.strict,
+            options.header,
+        )
+
+
+def main() -> None:
+    try:
+        run(sys.argv, sys.stdin, sys.stdout)
+    except OSError as e:
+        # Workaround for close failed in file object destructor: sys.excepthook is missing lost sys.stderr
+        # http://stackoverflow.com/questions/7955138/addressing-sys-excepthook-error-in-bash-script
+        sys.stderr.write(str(e) + '\n')
+        sys.exit(1)
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+
+# @omlish-manifest
+_CLI_MODULE = CliModule('pawk', __name__)
+
+
+if __name__ == '__main__':
+    main()

{omdev-0.0.0.dev97 → omdev-0.0.0.dev98/omdev.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev97
+Version: 0.0.0.dev98
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,13 +12,14 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev97
+Requires-Dist: omlish==0.0.0.dev98
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"
-Requires-Dist: cffi~=1.17; extra == "all"
 Requires-Dist: pcpp~=1.30; extra == "all"
+Requires-Dist: cffi~=1.17; extra == "all"
 Requires-Dist: docutils~=0.21; extra == "all"
+Requires-Dist: markdown~=3.7; extra == "all"
 Requires-Dist: mypy~=1.11; extra == "all"
 Requires-Dist: gprof2dot~=2024.6; extra == "all"
 Requires-Dist: prompt-toolkit~=3.0; extra == "all"
@@ -28,10 +29,11 @@ Provides-Extra: black
 Requires-Dist: black~=24.10; extra == "black"
 Provides-Extra: c
 Requires-Dist: pycparser~=2.22; extra == "c"
-Requires-Dist: cffi~=1.17; extra == "c"
 Requires-Dist: pcpp~=1.30; extra == "c"
-Provides-Extra: docutils
-Requires-Dist: docutils~=0.21; extra == "docutils"
+Requires-Dist: cffi~=1.17; extra == "c"
+Provides-Extra: doc
+Requires-Dist: docutils~=0.21; extra == "doc"
+Requires-Dist: markdown~=3.7; extra == "doc"
 Provides-Extra: mypy
 Requires-Dist: mypy~=1.11; extra == "mypy"
 Provides-Extra: prof

{omdev-0.0.0.dev97 → omdev-0.0.0.dev98}/omdev.egg-info/SOURCES.txt

@@ -131,11 +131,14 @@ omdev/toml/parser.py
 omdev/toml/writer.py
 omdev/tools/__init__.py
 omdev/tools/dockertools.py
+omdev/tools/doctools.py
 omdev/tools/gittools.py
 omdev/tools/importscan.py
 omdev/tools/mkrelimp.py
 omdev/tools/nbtools.py
 omdev/tools/piptools.py
 omdev/tools/proftools.py
-omdev/tools/rsttool.py
-omdev/tools/sqlrepl.py
+omdev/tools/sqlrepl.py
+omdev/tools/pawk/__init__.py
+omdev/tools/pawk/__main__.py
+omdev/tools/pawk/pawk.py