omdev 0.0.0.dev162__tar.gz → 0.0.0.dev164__tar.gz

{omdev-0.0.0.dev162/omdev.egg-info → omdev-0.0.0.dev164}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev162
+Version: 0.0.0.dev164
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev162
+Requires-Dist: omlish==0.0.0.dev164
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev162 → omdev-0.0.0.dev164}/omdev/amalg/__main__.py

@@ -6,6 +6,6 @@ _CLI_MODULE = CliModule('amalg', __name__)
 
 
 if __name__ == '__main__':
-    from .amalg import _main
+    from .main import _main
 
     _main()

omdev-0.0.0.dev164/omdev/amalg/gen.py

@@ -0,0 +1,212 @@
+import io
+import os.path
+import textwrap
+import typing as ta
+
+from omlish import cached
+from omlish import collections as col
+from omlish import lang
+from omlish.lite.runtime import LITE_REQUIRED_PYTHON_VERSION
+
+from .. import tokens as tks
+from .srcfiles import SrcFile
+from .srcfiles import make_src_file
+from .strip import strip_main_lines
+from .types import Tokens
+from .typing import Typing
+
+
+##
+
+
+SECTION_SEP = '#' * 40 + '\n'
+
+RUFF_DISABLES: ta.AbstractSet[str] = {
+    'UP006',  # non-pep585-annotation
+    'UP007',  # non-pep604-annotation
+    'UP036',  # outdated-version-block
+}
+
+OUTPUT_COMMENT = '# @omlish-amalg-output '
+SCAN_COMMENT = '# @omlish-amalg '
+
+
+class AmalgGenerator:
+    def __init__(
+            self,
+            main_path: str,
+            *,
+            mounts: ta.Mapping[str, str],
+            output_dir: str | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._main_path = main_path
+        self._mounts = mounts
+        self._output_dir = output_dir
+
+    @cached.function
+    def _src_files(self) -> dict[str, SrcFile]:
+        src_files: dict[str, SrcFile] = {}
+        todo = [self._main_path]
+        while todo:
+            src_path = todo.pop()
+            if src_path in src_files:
+                continue
+
+            f = make_src_file(
+                src_path,
+                mounts=self._mounts,
+            )
+            src_files[src_path] = f
+
+            for imp in f.imports:
+                if (mp := imp.mod_path) is not None:
+                    todo.append(mp)
+
+        return src_files
+
+    @cached.function
+    def _main_file(self) -> SrcFile:
+        return self._src_files()[self._main_path]
+
+    @cached.function
+    def _header_lines(self) -> list[Tokens]:
+        header_lines = []
+
+        if self._main_file().header_lines:
+            header_lines.extend([
+                hl
+                for hlts in self._main_file().header_lines
+                if not (hl := tks.join_toks(hlts)).startswith(SCAN_COMMENT)
+            ])
+
+        if self._output_dir is not None:
+            ogf = os.path.relpath(self._main_path, self._output_dir)
+        else:
+            ogf = os.path.basename(self._main_path)
+
+        additional_header_lines = [
+            '#!/usr/bin/env python3\n',
+            '# noinspection DuplicatedCode\n',
+            '# @omlish-lite\n',
+            '# @omlish-script\n',
+            f'{OUTPUT_COMMENT.strip()} {ogf}\n',
+        ]
+
+        ruff_disables = sorted({
+            *lang.flatten(f.ruff_noqa for f in self._src_files().values()),
+            *RUFF_DISABLES,
+        })
+        if ruff_disables:
+            additional_header_lines.append(f'# ruff: noqa: {" ".join(sorted(ruff_disables))}\n')
+
+        return [*additional_header_lines, *header_lines]
+
+    @cached.function
+    def gen_amalg(self) -> str:
+        src_files = self._src_files()
+
+        ##
+
+        out = io.StringIO()
+
+        ##
+
+        out.write(''.join(self._header_lines()))
+
+        ##
+
+        all_imps = [i for f in src_files.values() for i in f.imports]
+        gl_imps = [i for i in all_imps if i.mod_path is None]
+
+        dct: dict = {
+            ('sys', None, None): ['import sys\n'],
+        }
+        if any(sf.has_binary_resources for sf in src_files.values()):
+            dct[('base64', None, None)] = ['import base64\n']
+        for imp in gl_imps:
+            dct.setdefault((imp.mod, imp.item, imp.as_), []).append(imp)
+        for _, l in sorted(dct.items()):
+            il = l[0]
+            out.write(il if isinstance(il, str) else tks.join_toks(il.toks))
+        if dct:
+            out.write('\n\n')
+
+        ##
+
+        out.write(SECTION_SEP)
+        out.write('\n\n')
+
+        version_check_fail_msg = (
+            f'Requires python {LITE_REQUIRED_PYTHON_VERSION!r}, '
+            f'got {{sys.version_info}} from {{sys.executable}}'
+        )
+        out.write(textwrap.dedent(f"""
+        if sys.version_info < {LITE_REQUIRED_PYTHON_VERSION!r}:
+            raise OSError(f{version_check_fail_msg!r})  # noqa
+        """).lstrip())
+        out.write('\n\n')
+
+        ##
+
+        ts = list(col.toposort({  # noqa
+            f.path: {mp for i in f.imports if (mp := i.mod_path) is not None}
+            for f in src_files.values()
+        }))
+        sfs = [sf for ss in ts for sf in sorted(ss)]
+
+        ##
+
+        tyd: dict[str, list[Typing]] = {}
+        tys = set()
+        for sf in sfs:
+            f = src_files[sf]
+            for ty in f.typings:
+                if ty.src not in tys:
+                    tyd.setdefault(f.path, []).append(ty)
+                    tys.add(ty.src)
+        if tys:
+            out.write(SECTION_SEP)
+            out.write('\n\n')
+        for i, (sf, ftys) in enumerate(tyd.items()):
+            f = src_files[sf]
+            if i:
+                out.write('\n')
+            if f is not self._main_file():
+                rp = os.path.relpath(f.path, os.path.dirname(self._main_file().path))
+            else:
+                rp = os.path.basename(f.path)
+            out.write(f'# {rp}\n')
+            for ty in ftys:
+                out.write(ty.src)
+        if tys:
+            out.write('\n\n')
+
+        ##
+
+        main_file = self._main_file()
+        for i, sf in enumerate(sfs):
+            f = src_files[sf]
+            out.write(SECTION_SEP)
+            if f is not main_file:
+                rp = os.path.relpath(f.path, main_file.path)
+            else:
+                rp = os.path.basename(f.path)
+            out.write(f'# {rp}\n')
+            if f is not main_file and f.header_lines:
+                out.write(tks.join_lines(f.header_lines))
+            out.write(f'\n\n')
+            cls = f.content_lines
+            if f is not main_file:
+                cls = strip_main_lines(cls)
+            sf_src = tks.join_lines(cls)
+            out.write(sf_src.strip())
+            if i < len(sfs) - 1:
+                out.write('\n\n\n')
+            else:
+                out.write('\n')
+
+        ##
+
+        return out.getvalue()

omdev-0.0.0.dev164/omdev/amalg/imports.py

@@ -0,0 +1,98 @@
+import dataclasses as dc
+import os.path
+import typing as ta
+
+from omlish import check
+
+from .. import tokens as tks
+from .types import Tokens
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Import:
+    mod: str
+    item: str | None
+    as_: str | None
+
+    src_path: str
+    line: int
+
+    mod_path: str | None
+
+    toks: Tokens = dc.field(repr=False)
+
+
+def make_import(
+        lts: Tokens,
+        *,
+        src_path: str,
+        mounts: ta.Mapping[str, str],
+) -> Import | None:
+    if not lts:
+        return None
+    ft = lts[0]
+
+    if ft.name != 'NAME' or ft.src not in ('import', 'from'):
+        return None
+
+    ml = []
+    il: list[str] | None = None
+    as_ = None
+    for tok in (it := iter(tks.ignore_ws(lts[1:]))):
+        if tok.name in ('NAME', 'OP'):
+            if tok.src == 'as':
+                check.none(as_)
+                nt = next(it)
+                check.equal(nt.name, 'NAME')
+                as_ = nt.src
+            elif tok.src == 'import':
+                check.equal(ft.src, 'from')
+                il = []
+            elif il is not None:
+                il.append(tok.src)
+            else:
+                ml.append(tok.src)
+        else:
+            raise Exception(tok)
+
+    mod = ''.join(ml)
+    item = ''.join(il) if il is not None else None
+
+    if (mnt := mounts.get(mod.partition('.')[0])) is not None:
+        ps = mod.split('.')
+        mod_path = os.path.abspath(os.path.join(
+            mnt,
+            *ps[1:-1],
+            ps[-1] + '.py',
+        ))
+
+    elif not mod.startswith('.'):
+        mod_path = None
+
+    else:
+        parts = mod.split('.')
+        nd = len(parts) - parts[::-1].index('')
+        mod_path = os.path.abspath(os.path.join(
+            os.path.dirname(src_path),
+            '../' * (nd - 1),
+            *parts[nd:-1],
+            parts[-1] + '.py',
+        ))
+
+        mod = check.isinstance(mod_path, str)
+
+    return Import(
+        mod=mod,
+        item=item,
+        as_=as_,
+
+        src_path=src_path,
+        line=ft.line,
+
+        mod_path=mod_path,
+
+        toks=lts,
+    )

omdev-0.0.0.dev164/omdev/amalg/main.py

@@ -0,0 +1,154 @@
+"""
+Conventions:
+ - must import whole global modules, if aliased must all match
+ - must import 'from' items for local modules
+
+TODO:
+ - !! check only importing lite code
+ - !! strip manifests? or relegate them to a separate tiny module ala __main__.py?
+  - # @omlish-no-amalg ? in cli.types? will strip stmt (more than 1 line) following @manifest, so shouldn't import
+ - more sanity checks lol
+ - typealias - support # noqa, other comments, and lamely support multiline by just stealing lines till it parses
+ - remove `if __name__ == '__main__':` blocks - thus, convention: no def _main() for these
+
+See:
+ - https://github.com/xonsh/amalgamate - mine is for portability not speed, and doesn't try to work on unmodified code
+
+Targets:
+ - interp
+ - pyproject
+ - precheck
+ - build
+ - pyremote
+ - bootstrap
+ - deploy
+ - supervisor?
+"""
+import argparse
+import logging
+import os.path
+import typing as ta
+
+from omlish import check
+from omlish.logs import all as logs
+
+from .gen import SCAN_COMMENT
+from .gen import AmalgGenerator
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+def _gen_one(
+        input_path: str,
+        output_path: str | None,
+        *,
+        mounts: ta.Mapping[str, str],
+) -> None:
+    log.info('Generating: %s -> %s', input_path, output_path)
+
+    src = AmalgGenerator(
+        input_path,
+        mounts=mounts,
+        output_dir=os.path.dirname(output_path if output_path is not None else input_path),
+    ).gen_amalg()
+
+    if output_path is not None:
+        with open(output_path, 'w') as f:
+            f.write(src)
+        os.chmod(output_path, os.stat(input_path).st_mode)
+
+    else:
+        print(src)
+
+
+def _scan_one(
+        input_path: str,
+        **kwargs: ta.Any,
+) -> None:
+    if not input_path.endswith('.py'):
+        return
+
+    with open(input_path) as f:
+        src = f.read()
+
+    sls = [l for l in src.splitlines() if l.startswith(SCAN_COMMENT)]
+    for sl in sls:
+        sas = sl[len(SCAN_COMMENT):].split()
+        if len(sas) != 1:
+            raise Exception(f'Invalid scan args: {input_path=} {sas=}')
+
+        output_path = os.path.abspath(os.path.join(os.path.dirname(input_path), sas[0]))
+        _gen_one(
+            input_path,
+            output_path,
+            **kwargs,
+        )
+
+
+def _gen_cmd(args) -> None:
+    if not os.path.isfile('pyproject.toml'):
+        raise Exception('Not in project root')
+
+    mounts = {}
+    for m in args.mounts or ():
+        if ':' not in m:
+            mounts[m] = os.path.abspath(m)
+        else:
+            k, v = m.split(':')
+            mounts[k] = os.path.abspath(v)
+
+    for i in args.inputs:
+        if os.path.isdir(i):
+            log.info('Scanning %s', i)
+            for we_dirpath, we_dirnames, we_filenames in os.walk(i):  # noqa
+                for fname in we_filenames:
+                    _scan_one(
+                        os.path.abspath(os.path.join(we_dirpath, fname)),
+                        mounts=mounts,
+                    )
+
+        else:
+            output_dir = args.output
+            if output_dir is not None:
+                output_path = check.isinstance(os.path.join(output_dir, os.path.basename(i)), str)
+            else:
+                output_path = None
+
+            _gen_one(
+                os.path.abspath(i),
+                output_path,
+                mounts=mounts,
+            )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    parser_gen = subparsers.add_parser('gen')
+    parser_gen.add_argument('--mount', '-m', dest='mounts', action='append')
+    parser_gen.add_argument('--output', '-o')
+    parser_gen.add_argument('inputs', nargs='+')
+    parser_gen.set_defaults(func=_gen_cmd)
+
+    return parser
+
+
+def _main() -> None:
+    logs.configure_standard_logging('INFO')
+
+    parser = _build_parser()
+    args = parser.parse_args()
+    if not getattr(args, 'func', None):
+        parser.print_help()
+    else:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    _main()

omdev-0.0.0.dev164/omdev/amalg/manifests.py

@@ -0,0 +1,49 @@
+import ast
+
+import tokenize_rt as trt
+
+from omlish import check
+
+from .. import tokens as tks
+from .types import Tokens
+
+
+##
+
+
+def is_manifest_comment(line: Tokens) -> bool:
+    if not line:
+        return False
+
+    return (
+        (ft := line[0]).name == 'COMMENT' and
+        ft.src.startswith('# @omlish-manifest')
+    )
+
+
+def comment_out_manifest_comment(
+        line: Tokens,
+        cls: list[Tokens],
+        i: int,
+) -> tuple[list[Tokens], int]:
+    mls = [line]
+    while True:
+        mls.append(cls[i])
+        i += 1
+
+        msrc = tks.join_lines(mls).strip()
+        try:
+            node = ast.parse(msrc)
+        except SyntaxError:
+            continue
+
+        mmod = check.isinstance(node, ast.Module)
+        check.isinstance(check.single(mmod.body), ast.Assign)
+        break
+
+    out: list[Tokens] = [
+        [trt.Token('COMMENT', '# ' + tks.join_toks(ml))]
+        for ml in mls
+    ]
+
+    return out, i

omdev-0.0.0.dev164/omdev/amalg/resources.py

@@ -0,0 +1,99 @@
+import ast
+import base64
+import itertools
+import os.path
+import typing as ta
+
+import tokenize_rt as trt
+
+from .. import tokens as tks
+from .types import Tokens
+
+
+##
+
+
+class RootLevelResourcesRead(ta.NamedTuple):
+    variable: str
+    kind: ta.Literal['binary', 'text']
+    resource: str
+
+
+def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
+    wts = list(tks.ignore_ws(lts, keep=['INDENT']))
+
+    if not tks.match_toks(wts, [
+        ('NAME', None),
+        ('OP', '='),
+        ('NAME', ('read_package_resource_binary', 'read_package_resource_text')),
+        ('OP', '('),
+        ('NAME', '__package__'),
+        ('OP', ','),
+        ('STRING', None),
+        ('OP', ')'),
+    ]):
+        return None
+
+    return RootLevelResourcesRead(
+        wts[0].src,
+        'binary' if wts[2].src == 'read_package_resource_binary' else 'text',
+        ast.literal_eval(wts[6].src),
+    )
+
+
+##
+
+
+def build_resource_lines(
+        rsrc: RootLevelResourcesRead,
+        path: str,
+) -> list[Tokens]:
+    rf = os.path.join(os.path.dirname(path), rsrc.resource)
+
+    if rsrc.kind == 'binary':
+        with open(rf, 'rb') as bf:
+            rb = bf.read()  # noqa
+
+        out: list[Tokens] = [[
+            trt.Token(name='NAME', src=rsrc.variable),
+            trt.Token(name='UNIMPORTANT_WS', src=' '),
+            trt.Token(name='OP', src='='),
+            trt.Token(name='UNIMPORTANT_WS', src=' '),
+            trt.Token(name='NAME', src='base64'),
+            trt.Token(name='OP', src='.'),
+            trt.Token(name='NAME', src='b64decode'),
+            trt.Token(name='OP', src='('),
+            trt.Token(name='NL', src='\n'),
+        ]]
+
+        rb64 = base64.b64encode(rb).decode('ascii')
+        for chunk in itertools.batched(rb64, 96):
+            out.append([
+                trt.Token(name='UNIMPORTANT_WS', src='    '),
+                trt.Token(name='STRING', src=f"'{''.join(chunk)}'"),
+                trt.Token(name='NL', src='\n'),
+            ])
+
+        out.append([
+            trt.Token(name='OP', src=')'),
+            trt.Token(name='NEWLINE', src='\n'),
+        ])
+
+        return out
+
+    elif rsrc.kind == 'text':
+        with open(rf) as tf:
+            rt = tf.read()  # noqa
+        rt = rt.replace('\\', '\\\\')  # Escape backslashes
+        rt = rt.replace('"""', r'\"\"\"')
+        return [[
+            trt.Token(name='NAME', src=rsrc.variable),
+            trt.Token(name='UNIMPORTANT_WS', src=' '),
+            trt.Token(name='OP', src='='),
+            trt.Token(name='UNIMPORTANT_WS', src=' '),
+            trt.Token(name='STRING', src=f'"""\\\n{rt}"""\n'),
+            trt.Token(name='NEWLINE', src=''),
+        ]]
+
+    else:
+        raise ValueError(rsrc.kind)