omdev 0.0.0.dev124__tar.gz → 0.0.0.dev137__tar.gz

{omdev-0.0.0.dev124/omdev.egg-info → omdev-0.0.0.dev137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev124
+Version: 0.0.0.dev137
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev124
+Requires-Dist: omlish==0.0.0.dev137
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev124 → omdev-0.0.0.dev137}/omdev/.manifests.json

@@ -95,6 +95,21 @@
       }
     }
   },
+  {
+    "module": ".json.__main__",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/json/__main__.py",
+    "line": 4,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": [
+          "json",
+          "j"
+        ],
+        "mod_name": "omdev.json.__main__"
+      }
+    }
+  },
   {
     "module": ".magic.find",
     "attr": "_CLI_MODULE",
@@ -243,7 +258,7 @@
     "module": ".tools.git",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/git.py",
-    "line": 205,
+    "line": 211,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "git",

{omdev-0.0.0.dev124 → omdev-0.0.0.dev137}/omdev/amalg/amalg.py

@@ -222,22 +222,30 @@ class Typing:
     toks: Tokens = dc.field(repr=False)
 
 
-def _is_typing(lts: Tokens) -> bool:
+def _is_typing(
+        lts: Tokens,
+        *,
+        exclude_newtypes: bool = False,
+) -> bool:
     es = tks.join_toks(lts).strip()
     if any(es.endswith(sfx) for sfx in (TYPE_ALIAS_COMMENT, NOQA_TYPE_ALIAS_COMMENT)):
         return True
 
     wts = list(tks.ignore_ws(lts))
-    if (
-            len(wts) >= 5 and
-            wts[0].name == 'NAME' and
-            wts[1].name == 'OP' and wts[1].src == '=' and
-            wts[2].name == 'NAME' and wts[2].src == 'ta' and
-            wts[3].name == 'OP' and wts[3].src == '.'
-    ):
-        return True
+    if not tks.match_toks(wts, [
+        ('NAME', None),
+        ('OP', '='),
+        ('NAME', 'ta'),
+        ('OP', '.'),
+        (None, None),
+    ]):
+        return False
+
+    if exclude_newtypes:
+        if wts[4].name == 'NAME' and wts[4].src == 'NewType':
+            return False
 
-    return False
+    return True
 
 
 def make_typing(
@@ -248,7 +256,7 @@ def make_typing(
     if not lts or lts[0].name == 'UNIMPORTANT_WS':
         return None
 
-    if not _is_typing(lts):
+    if not _is_typing(lts, exclude_newtypes=True):
         return None
 
     ft = next(iter(tks.ignore_ws(lts)))
@@ -265,6 +273,19 @@ def make_typing(
 ##
 
 
+def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
+    return tks.match_toks(tks.ignore_ws(lts, keep=['INDENT']), [
+        ('NAME', 'if'),
+        ('NAME', 'ta'),
+        ('OP', '.'),
+        ('NAME', 'TYPE_CHECKING'),
+        ('OP', ':'),
+    ])
+
+
+##
+
+
 @dc.dataclass(frozen=True, kw_only=True)
 class SrcFile:
     path: str
@@ -344,6 +365,13 @@ def make_src_file(
                 for ml in mls
             ])
 
+        elif is_root_level_if_type_checking_block(line):
+            while True:
+                nl = cls[i]
+                if nl and nl[0].name != 'INDENT':
+                    break
+                i += 1
+
         else:
             ctls.append(line)
 
@@ -498,7 +526,7 @@ def gen_amalg(
         if i:
             out.write('\n')
         if f is not mf:
-            rp = os.path.relpath(f.path, mf.path)
+            rp = os.path.relpath(f.path, os.path.dirname(mf.path))
         else:
             rp = os.path.basename(f.path)
         out.write(f'# {rp}\n')

{omdev-0.0.0.dev124 → omdev-0.0.0.dev137}/omdev/cexts/_distutils/sysconfig.py

@@ -73,7 +73,7 @@ def get_python_version():
     Return a string containing the major and minor Python version, leaving off the patchlevel.  Sample return values
     could be '1.5' or '2.2'.
     """
-    return '%d.%d' % sys.version_info[:2]
+    return '%d.%d' % sys.version_info[:2]  # noqa
 
 
 def get_python_inc(plat_specific=0, prefix=None):
@@ -193,9 +193,9 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
 
     if prefix is None:
         if standard_lib:
-            prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX
+            prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX  # noqa
         else:
-            prefix = plat_specific and EXEC_PREFIX or PREFIX
+            prefix = plat_specific and EXEC_PREFIX or PREFIX  # noqa
 
     if os.name == 'posix':
         if plat_specific or standard_lib:

{omdev-0.0.0.dev124 → omdev-0.0.0.dev137}/omdev/cli/clicli.py

@@ -21,11 +21,11 @@ class CliCli(ap.Cli):
 
     #
 
-    @ap.command(name='version')
+    @ap.command(name='version', aliases=['ver'])
     def print_version(self) -> None:
         print(__about__.__version__)
 
-    @ap.command(name='revision')
+    @ap.command(name='revision', aliases=['rev'])
     def print_revision(self) -> None:
         print(__about__.__revision__)
 

omdev-0.0.0.dev137/omdev/json/__main__.py

@@ -0,0 +1,11 @@
+from ..cli import CliModule
+
+
+# @omlish-manifest
+_CLI_MODULE = CliModule(['json', 'j'], __name__)
+
+
+if __name__ == '__main__':
+    from .cli import _main
+
+    _main()

omdev-0.0.0.dev137/omdev/json/cli.py

@@ -0,0 +1,299 @@
+"""
+TODO:
+ - read from http
+ - jmespath output flat, unquoted strs like jq '.[]'
+
+==
+
+jq Command options:
+  -n, --null-input          use `null` as the single input value;
+  -R, --raw-input           read each line as string instead of JSON;
+  -s, --slurp               read all inputs into an array and use it as the single input value;
+  -c, --compact-output      compact instead of pretty-printed output;
+  -r, --raw-output          output strings without escapes and quotes;
+      --raw-output0         implies -r and output NUL after each output;
+  -j, --join-output         implies -r and output without newline after each output;
+  -a, --ascii-output        output strings by only ASCII characters using escape sequences;
+  -S, --sort-keys           sort keys of each object on output;
+  -C, --color-output        colorize JSON output;
+  -M, --monochrome-output   disable colored output;
+      --tab                 use tabs for indentation;
+      --indent n            use n spaces for indentation (max 7 spaces);
+      --unbuffered          flush output stream after each output;
+      --stream              parse the input value in streaming fashion;
+      --stream-errors       implies --stream and report parse error as an array;
+      --seq                 parse input/output as application/json-seq;
+  -f, --from-file file      load filter from the file;
+  -L directory              search modules from the directory;
+      --arg name value      set $name to the string value;
+      --argjson name value  set $name to the JSON value;
+      --slurpfile name file set $name to an array of JSON values read from the file;
+      --rawfile name file   set $name to string contents of file;
+      --args                consume remaining arguments as positional string values;
+      --jsonargs            consume remaining arguments as positional JSON values;
+  -e, --exit-status         set exit status code based on the output;
+  -V, --version             show the version;
+  --build-configuration     show jq's build configuration;
+  -h, --help                show the help;
+  --                        terminates argument processing;
+"""
+import argparse
+import contextlib
+import dataclasses as dc
+import io
+import os
+import subprocess
+import sys
+import typing as ta
+
+from omlish import check
+from omlish import lang
+from omlish.funcs import pipes as fp
+
+from .formats import FORMATS_BY_NAME
+from .formats import Format
+from .formats import Formats
+from .parsing import DelimitingParser
+from .parsing import EagerParser
+from .parsing import StreamBuilder
+from .parsing import StreamParser
+from .processing import ProcessingOptions
+from .processing import Processor
+from .rendering import EagerRenderer
+from .rendering import RenderingOptions
+from .rendering import StreamRenderer
+
+
+T = ta.TypeVar('T')
+U = ta.TypeVar('U')
+
+
+def _build_args_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('file', nargs='?')
+
+    parser.add_argument('--stream', action='store_true')
+    parser.add_argument('--stream-build', action='store_true')
+
+    parser.add_argument('-l', '--lines', action='store_true')
+
+    parser.add_argument('--read-buffer-size', type=int, default=0x4000)
+
+    parser.add_argument('-f', '--format')
+
+    parser.add_argument('-x', '--jmespath-expr')
+    parser.add_argument('-F', '--flat', action='store_true')
+
+    parser.add_argument('-z', '--compact', action='store_true')
+    parser.add_argument('-p', '--pretty', action='store_true')
+    parser.add_argument('-i', '--indent')
+    parser.add_argument('-s', '--sort-keys', action='store_true')
+    parser.add_argument('-R', '--raw', action='store_true')
+    parser.add_argument('-U', '--unicode', action='store_true')
+    parser.add_argument('-c', '--color', action='store_true')
+
+    parser.add_argument('-L', '--less', action='store_true')
+
+    return parser
+
+
+def _parse_args(args: ta.Any = None) -> ta.Any:
+    return _build_args_parser().parse_args(args)
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class RunConfiguration:
+    format: Format
+    processing: ProcessingOptions
+    rendering: RenderingOptions
+
+
+def _process_args(args: ta.Any) -> RunConfiguration:
+    fmt_name = args.format
+    if fmt_name is None:
+        if args.file is not None:
+            ext = args.file.rpartition('.')[2]
+            if ext in FORMATS_BY_NAME:
+                fmt_name = ext
+    if fmt_name is None:
+        fmt_name = 'json'
+    format = FORMATS_BY_NAME[fmt_name]  # noqa
+
+    if args.stream:
+        check.arg(format is Formats.JSON.value)
+
+    #
+
+    processing = ProcessingOptions(
+        jmespath_expr=args.jmespath_expr,
+        flat=args.flat,
+    )
+
+    #
+
+    separators = None
+    if args.compact:
+        separators = (',', ':')
+
+    indent = None
+    if args.pretty:
+        indent = 2
+    if args.indent:
+        try:
+            indent = int(args.indent)
+        except ValueError:
+            indent = args.indent
+
+    rendering = RenderingOptions(
+        indent=indent,
+        separators=separators,
+        sort_keys=args.sort_keys,
+        raw=args.raw,
+        unicode=args.unicode,
+        color=args.color,
+    )
+
+    #
+
+    return RunConfiguration(
+        format=format,
+        processing=processing,
+        rendering=rendering,
+    )
+
+
+def _main() -> None:
+    args = _parse_args()
+
+    #
+
+    cfg = _process_args(args)
+
+    #
+
+    with contextlib.ExitStack() as es:
+        if args.file is None:
+            in_file = sys.stdin.buffer
+
+        else:
+            in_file = es.enter_context(open(args.file, 'rb'))
+
+        def yield_input() -> ta.Generator[bytes, None, None]:
+            fd = check.isinstance(in_file.fileno(), int)
+
+            while True:
+                buf = os.read(fd, args.read_buffer_size)
+
+                yield buf
+
+                if not buf:
+                    break
+
+        #
+
+        if args.less:
+            less = subprocess.Popen(
+                [
+                    'less',
+                    *(['-R'] if cfg.rendering.color else []),
+                ],
+                stdin=subprocess.PIPE,
+                encoding='utf-8',
+            )
+            out = check.not_none(less.stdin)
+
+            def close_less() -> None:
+                out.close()
+                less.wait()
+
+            es.enter_context(lang.defer(close_less))  # noqa
+
+        else:
+            out = sys.stdout
+
+        #
+
+        parser: ta.Any
+        renderer: ta.Any
+
+        if args.stream:
+            with contextlib.ExitStack() as es2:
+                parser = es2.enter_context(StreamParser())
+
+                def flush_output(
+                        fn: ta.Callable[[T], ta.Iterable[U]],
+                        i: T,
+                ) -> ta.Generator[U, None, None]:
+                    n = 0
+                    for o in fn(i):
+                        yield o
+                        n += 1
+                    if n:
+                        out.flush()
+
+                pipeline: ta.Any
+
+                if args.stream_build:
+                    builder: StreamBuilder = es2.enter_context(StreamBuilder())
+                    processor = Processor(cfg.processing)
+                    renderer = EagerRenderer(cfg.rendering)
+                    trailing_newline = False
+
+                    def append_newlines(
+                            fn: ta.Callable[[T], ta.Iterable[str]],
+                            i: T,
+                    ) -> ta.Generator[str, None, None]:
+                        yield from fn(i)
+                        yield '\n'
+
+                    pipeline = lambda v: (renderer.render(v),)  # Any -> [str]  # noqa
+                    pipeline = fp.bind(append_newlines, pipeline)  # Any -> [str]
+                    pipeline = fp.bind(lang.flatmap, pipeline)  # [Any] -> [str]
+                    pipeline = fp.pipe(fp.bind(lang.flatmap, processor.process), pipeline)  # [Any] -> [str]
+                    pipeline = fp.pipe(fp.bind(lang.flatmap, builder.build), pipeline)  # [JsonStreamParserEvent] -> [str]  # noqa
+                    pipeline = fp.pipe(parser.parse, pipeline)  # bytes -> [str]
+
+                else:
+                    renderer = StreamRenderer(cfg.rendering)
+                    trailing_newline = True
+
+                    pipeline = renderer.render  # JsonStreamParserEvent -> [str]
+                    pipeline = fp.bind(lang.flatmap, pipeline)  # [JsonStreamParserEvent] -> [str]
+                    pipeline = fp.pipe(parser.parse, pipeline)  # bytes -> [str]
+
+                pipeline = fp.bind(flush_output, pipeline)  # bytes -> [str]
+
+                for buf in yield_input():
+                    for s in pipeline(buf):
+                        print(s, file=out, end='')
+
+                if trailing_newline:
+                    print(file=out)
+
+        elif args.lines:
+            parser = DelimitingParser(cfg.format)
+            processor = Processor(cfg.processing)
+            renderer = EagerRenderer(cfg.rendering)
+
+            for buf in yield_input():
+                for v in parser.parse(buf):
+                    for e in processor.process(v):
+                        s = renderer.render(e)
+                        print(s, file=out)
+
+        else:
+            parser = EagerParser(cfg.format)
+            processor = Processor(cfg.processing)
+            renderer = EagerRenderer(cfg.rendering)
+
+            with io.TextIOWrapper(in_file) as tf:
+                v = parser.parse(tf)
+
+            for e in processor.process(v):
+                s = renderer.render(e)
+                print(s, file=out)
+
+
+if __name__ == '__main__':
+    _main()

omdev-0.0.0.dev137/omdev/json/formats.py

@@ -0,0 +1,71 @@
+"""
+TODO:
+ - options lol - csv header, newline, etc
+"""
+import dataclasses as dc
+import enum
+import json
+import typing as ta
+
+from omlish import lang
+
+
+if ta.TYPE_CHECKING:
+    import ast
+    import csv
+    import tomllib
+
+    import yaml
+
+    from omlish.formats import dotenv
+    from omlish.formats import props
+    from omlish.formats import xml
+
+else:
+    ast = lang.proxy_import('ast')
+    csv = lang.proxy_import('csv')
+    tomllib = lang.proxy_import('tomllib')
+
+    yaml = lang.proxy_import('yaml')
+
+    dotenv = lang.proxy_import('omlish.formats.dotenv')
+    props = lang.proxy_import('omlish.formats.props')
+    xml = lang.proxy_import('omlish.formats.xml')
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class Format:
+    names: ta.Sequence[str]
+    load: ta.Callable[[ta.TextIO], ta.Any]
+
+
+class Formats(enum.Enum):
+    JSON = Format(['json'], json.load)
+
+    YAML = Format(['yaml', 'yml'], lambda f: yaml.safe_load(f))
+
+    TOML = Format(['toml'], lambda f: tomllib.loads(f.read()))
+
+    ENV = Format(['env', 'dotenv'], lambda f: dotenv.dotenv_values(stream=f))
+
+    PROPS = Format(['properties', 'props'], lambda f: dict(props.Properties().load(f.read())))
+
+    PY = Format(['py', 'python', 'repr'], lambda f: ast.literal_eval(f.read()))
+
+    XML = Format(['xml'], lambda f: xml.build_simple_element(xml.parse_tree(f.read()).getroot()).as_dict())
+
+    CSV = Format(['csv'], lambda f: list(csv.DictReader(f)))
+    TSV = Format(['tsv'], lambda f: list(csv.DictReader(f, delimiter='\t')))
+    FLAT_CSV = Format(['fcsv'], lambda f: list(csv.reader(f)))
+    FLAT_TSV = Format(['ftsv'], lambda f: list(csv.reader(f, delimiter='\t')))
+
+
+FORMATS_BY_NAME: ta.Mapping[str, Format] = {
+    n: f
+    for e in Formats
+    for f in [e.value]
+    for n in f.names
+}

omdev-0.0.0.dev137/omdev/json/io.py

@@ -0,0 +1,74 @@
+# """
+# TODO:
+#  - -I/-Ogz, lz4, etc
+#   - fnpairs? or not yet just do it
+# """
+# import abc
+# import contextlib
+# import dataclasses as dc
+# import gzip
+# import os
+# import typing as ta
+#
+# from .... import lang
+#
+#
+# if ta.TYPE_CHECKING:
+#     import bz2 as _bz2
+#     import gzip as _gzip
+#     import lzma as _lzma
+# else:
+#     _bz2 = lang.proxy_import('bz2')
+#     _gzip = lang.proxy_import('gzip')
+#     _lzma = lang.proxy_import('lzma')
+#
+#
+# ##
+#
+#
+# class BaseIo(lang.Abstract):
+#     def close(self) -> None:
+#         raise NotImplementedError
+#
+#     def fileno(self) -> int | None:
+#         return None
+#
+#
+# class Input(BaseIo):
+#     @abc.abstractmethod
+#     def read(self, sz: int | None = None) -> bytes:
+#         raise NotImplementedError
+#
+#
+# class Output(BaseIo):
+#     @abc.abstractmethod
+#     def write(self, data: bytes) -> int:
+#         raise NotImplementedError
+#
+#
+# #
+#
+#
+# DEFAULT_READ_SZ = 0x4000
+#
+#
+# @dc.dataclass(frozen=True)
+# class FdIo(Input, Output):
+#     fd: int
+#
+#     default_read_sz: int = DEFAULT_READ_SZ
+#
+#     def read(self, sz: int | None = None) -> bytes:
+#         return os.read(self.fd, sz or self.default_read_sz)
+#
+#     def write(self, data: bytes) -> int:
+#         return os.write(self.fd, data)
+#
+#
+# ##
+#
+#
+# @contextlib.contextmanager
+# def gzip_io_codec(f: ta.IO, mode: str) -> ta.ContextManager[ta.IO]:
+#     with gzip.open(f, mode) as o:
+#         yield o