omdev 0.0.0.dev440__py3-none-any.whl → 0.0.0.dev495__py3-none-any.whl

omdev/dataclasses/_template.py

@@ -0,0 +1,22 @@
+# type: ignore
+# ruff: noqa
+# flake8: noqa
+import dataclasses
+import reprlib
+import types
+
+
+##
+
+
+REGISTRY = {}
+
+
+def _register(**kwargs):
+    def inner(fn):
+        REGISTRY[kwargs['plan_repr']] = (kwargs, fn)
+        return fn
+    return inner
+
+
+##

omdev/dataclasses/cli.py

@@ -1,5 +1,5 @@
 from omlish.argparse import all as ap
-from omlish.logs.standard import configure_standard_logging
+from omlish.logs.std.standard import configure_standard_logging
 
 from .codegen import DataclassCodeGen
 
@@ -12,7 +12,12 @@ class Cli(ap.Cli):
         ap.arg('roots', metavar='root', nargs='+'),
     )
     def codegen(self) -> None:
-        DataclassCodeGen().run(self.args.roots)
+        import asyncio
+        asyncio.run(DataclassCodeGen(
+            # dump_inline=True,
+        ).run(
+            self.args.roots,
+        ))
 
 
 ##

omdev/dataclasses/codegen.py

@@ -1,27 +1,38 @@
 """
 TODO:
- - subdir conf files override parents, codegen those separately, don't duplicate
  - refactor dc gen to just Execute and Codegen
  - need to bubble up imports, preamble, deduped
- - still need plan repr / cmp
- - !! manifests for dataclass config?
-  - more sparse / diffuse intent, not package-level
+ - _gen subdirs
+ - static analyze codegen kwarg if possible
+ - better ignore configurability than just tests dirs lol
 """
-import importlib
+import ast
+import asyncio
+import hashlib
+import inspect
+import json
 import os.path
+import shlex
+import sys
+import tempfile
+import time
 import typing as ta
 
 from omlish import check
 from omlish import collections as col
+from omlish import dataclasses as dc
 from omlish import lang
-from omlish.dataclasses.impl.configs import PackageConfig
-from omlish.dataclasses.impl.generation.compilation import OpCompiler
-from omlish.dataclasses.impl.generation.processor import Codegen as CodegenProcessingOption
-from omlish.dataclasses.impl.generation.processor import GeneratorProcessor
-from omlish.dataclasses.impl.processing.base import ProcessingContext
-from omlish.dataclasses.impl.processing.driving import processing_options_context
+from omlish.asyncs.asyncio import all as au
+from omlish.lite.marshal import unmarshal_obj
 from omlish.logs import all as logs
 
+from ..py.asts.toplevel import TopLevelCall
+from ..py.asts.toplevel import analyze_module_top_level
+from ..py.reprs import textwrap_repr
+from ..py.srcheaders import get_py_header_lines
+from .dumping import DataclassCodegenDumperOutput
+from .dumping import DumpedDataclassCodegen
+
 
 log = logs.get_module_logger(globals())
 
@@ -29,67 +40,336 @@ log = logs.get_module_logger(globals())
 ##
 
 
+def _find_dir_py_files(dir_path: str) -> list[str]:
+    return sorted(
+        os.path.join(p, fn)
+        for p, dns, fns in os.walk(dir_path)
+        for fn in fns
+        if fn.endswith('.py')
+    )
+
+
+@lang.cached_function
+def _module_manifest_dumper_payload_src() -> str:
+    from . import _dumping
+    return inspect.getsource(_dumping)
+
+
+def _is_generated_py_file(fp: str) -> bool:
+    with open(fp) as f:
+        gen_file_src = f.read()
+
+    gen_hdr_lines = get_py_header_lines(gen_file_src)
+    return any(hl.src.strip() == '# @omlish-generated' for hl in gen_hdr_lines)
+
+
+#
+
+
 class DataclassCodeGen:
-    def __init__(self) -> None:
+    DEFAULT_TARGET_LINE_WIDTH: ta.ClassVar[int] = 120
+
+    def __init__(
+            self,
+            *,
+            target_line_width: int | None = None,
+            dump_inline: bool = False,
+            subprocess_kwargs: ta.Mapping[str, ta.Any] | None = None,
+    ) -> None:
         super().__init__()
 
-    def run_package_config(
+        self._target_line_width = target_line_width or self.DEFAULT_TARGET_LINE_WIDTH
+        self._dump_inline = dump_inline
+        self._subprocess_kwargs = subprocess_kwargs
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class ConfiguredPackage:
+        name: str
+
+        init_file_path: str
+        init_module_name: str
+
+        init_package_call: TopLevelCall
+
+    def scan_py_file(self, file_path: str) -> ConfiguredPackage | None:
+        with open(file_path) as f:
+            source = f.read()
+
+        module = check.isinstance(ast.parse(source), ast.Module)
+        module_name = '.'.join([
+            *os.path.dirname(file_path).split(os.path.sep),
+            os.path.basename(file_path).removesuffix('.py'),
+        ])
+
+        tl = analyze_module_top_level(module, module_name)
+
+        init_calls: list[TopLevelCall] = []
+        for call in tl.calls:
+            if call.imp.spec != 'omlish.dataclasses':
+                continue
+            match call.node:
+                case ast.Call(func=ast.Attribute(attr='init_package')):
+                    init_calls.append(call)
+
+        if not init_calls:
+            return None
+        if os.path.basename(file_path) != '__init__.py':
+            raise ValueError(f'File {file_path} has init_package call and is not an __init__.py: {init_calls!r}')
+        if len(init_calls) > 1:
+            raise ValueError(f'File {file_path} has multiple init_package calls: {init_calls!r}')
+
+        return DataclassCodeGen.ConfiguredPackage(
+            '.'.join(module_name.split('.')[:-1]),
+            file_path,
+            module_name,
+            check.single(init_calls),
+        )
+
+    def find_configured_packages(self, root_dirs: ta.Iterable[str]) -> list[ConfiguredPackage]:
+        check.not_isinstance(root_dirs, str)
+
+        py_files = (
+            fp
+            for rd in root_dirs
+            for fp in _find_dir_py_files(rd)
+        )
+
+        return [
+            cfg_pkg
+            for file_path in py_files
+            if (cfg_pkg := self.scan_py_file(file_path)) is not None
+        ]
+
+    #
+
+    async def _run_dumper_subprocess(
             self,
-            pkg_root: str,
-            config: PackageConfig,
+            cfg_pkg: ConfiguredPackage,
+            dumper_kwargs: ta.Mapping[str, ta.Any],
+            *,
+            shell_wrap: bool = True,
     ) -> None:
-        if not config.codegen:
-            return
-
-        log.info('Running codegen on package: %s', pkg_root)
-
-        sub_pkgs = sorted(lang.yield_importable(
-            pkg_root,
-            recursive=True,
-        ))
-
-        for sub_pkg in sub_pkgs:
-            def callback(
-                    ctx: ProcessingContext,
-                    prepared: GeneratorProcessor.Prepared,
-                    comp: OpCompiler.CompileResult,
-            ) -> None:
-                print(ctx.cls)
-                print(prepared.plans)
-                print(comp.src)
-
-            with processing_options_context(CodegenProcessingOption(callback)):
-                print(f'{sub_pkg=}')
-                try:
-                    importlib.import_module(sub_pkg)
-                except ImportError as e:
-                    print(repr(e))
-
-    def build_config_trie(
+        dumper_payload_src = _module_manifest_dumper_payload_src()
+
+        subproc_src = '\n\n'.join([
+            dumper_payload_src,
+            f'_DataclassCodegenDumper()(**{dumper_kwargs!r})\n',
+        ])
+
+        args = [
+            sys.executable,
+            '-c',
+            subproc_src,
+        ]
+
+        if shell_wrap:
+            args = ['sh', '-c', ' '.join(map(shlex.quote, args))]
+
+        proc = await asyncio.create_subprocess_exec(
+            *args,
+            **(self._subprocess_kwargs or {}),
+        )
+
+        if await proc.wait():
+            raise Exception('Subprocess failed')
+
+    async def _run_dumper_inline(
             self,
-            root_dirs: ta.Iterable[str],
-    ) -> col.Trie[str, PackageConfig]:
-        check.not_isinstance(root_dirs, str)
+            cfg_pkg: ConfiguredPackage,
+            dumper_kwargs: ta.Mapping[str, ta.Any],
+    ) -> None:
+        from . import dumping
+
+        dumping._DataclassCodegenDumper()(**dumper_kwargs)  # noqa
+
+    async def process_configured_package(
+            self,
+            cfg_pkg: ConfiguredPackage,
+            *,
+            warn_threshold_s: float | None = 10.,
+    ) -> None:
+        log.info(lambda: f'Running codegen on package: {cfg_pkg.name}')
 
-        trie: col.Trie[str, PackageConfig] = col.Trie()
-        for root_dir in root_dirs:
-            for dp, _, fns in os.walk(root_dir):  # noqa
-                # if PACKAGE_CONFIG_FILE_NAME in fns:
-                #     with open(os.path.join(dp, PACKAGE_CONFIG_FILE_NAME)) as f:
-                #         config = PackageConfig.loads(f.read())
-                #     pkg_parts = dp.split(os.sep)
-                #     trie[pkg_parts] = config
-                pass
+        out_dir = tempfile.mkdtemp()
+        out_file_path = os.path.join(out_dir, 'output.json')
 
-        return trie
+        dumper_kwargs = dict(
+            init_file_path=cfg_pkg.init_file_path,
+            out_file_path=out_file_path,
+        )
 
-    def run(
+        start_time = time.time()
+
+        if self._dump_inline:
+            await self._run_dumper_inline(cfg_pkg, dumper_kwargs)
+        else:
+            await self._run_dumper_subprocess(cfg_pkg, dumper_kwargs)
+
+        end_time = time.time()
+
+        if warn_threshold_s is not None and (elapsed_time := (end_time - start_time)) >= warn_threshold_s:
+            log.warning('Dataclass codegen took a long time: %s, %.2f s', cfg_pkg.name, elapsed_time)
+
+        with open(out_file_path) as f:  # noqa
+            out_s = f.read()
+
+        output: DataclassCodegenDumperOutput = unmarshal_obj(json.loads(out_s), DataclassCodegenDumperOutput)
+
+        await self.process_dumper_output(cfg_pkg, output)
+
+    #
+
+    PROCESS_FN_NAME: ta.ClassVar[str] = '_process_dataclass'
+
+    async def process_dumper_output(
+            self,
+            cfg_pkg: ConfiguredPackage,
+            output: DataclassCodegenDumperOutput,
+    ) -> None:
+        gen_file_path = os.path.join(os.path.dirname(cfg_pkg.init_file_path), '_dataclasses.py')
+
+        if os.path.isfile(gen_file_path):
+            if not _is_generated_py_file(gen_file_path):
+                raise RuntimeError(f'Refusing to overwrite non-generated file: {gen_file_path!r}')
+
+            if not output.dumped:
+                os.unlink(gen_file_path)
+                return
+
+        #
+
+        lines = [
+            '# @omlish-generated',
+        ]
+
+        from . import _template
+        lines.extend(inspect.getsource(_template).strip().split('\n'))
+
+        #
+
+        processed_modules = set(output.processed_modules)
+
+        dumped_by_plan_repr: dict[str, list[DumpedDataclassCodegen]] = {}
+
+        seen_cls_name_tups: set[tuple[str, str]] = set()
+
+        for x in output.dumped:
+            if x.cls_module not in processed_modules:
+                continue
+
+            cls_name_tup = (x.cls_module, x.cls_qualname)
+            check.not_in(cls_name_tup, seen_cls_name_tups)
+            seen_cls_name_tups.add(cls_name_tup)
+
+            try:
+                lst = dumped_by_plan_repr[x.plan_repr]
+            except KeyError:
+                dumped_by_plan_repr[x.plan_repr] = [x]
+                continue
+
+            y = lst[0]
+            if set(x.refs) != set(y.refs):
+                raise RuntimeError(f'Mismatched refs: {x!r} != {y!r}')
+
+            lst.append(x)
+
+        #
+
+        # Sorted by first cls name for more stable diffs than say sha1
+        for grp in sorted(
+                dumped_by_plan_repr.values(),
+                key=lambda grp: sorted([(y.mod_name, y.cls_qualname) for y in grp])[0],
+        ):
+            x = grp[0]
+            pr = x.plan_repr
+            pr_sha1 = hashlib.sha1(pr.encode()).hexdigest()  # noqa
+
+            fn_name = f'{self.PROCESS_FN_NAME}__{pr_sha1}'
+
+            lines.extend(['', ''])
+
+            lines.append(
+                '@_register(',
+            )
+
+            lines.extend([
+                f'    plan_repr=(',
+                *[
+                    f'        {prl}'
+                    for prl in textwrap_repr(x.plan_repr, self._target_line_width - 8)
+                ],
+                f'    ),',
+                f'    plan_repr_sha1={pr_sha1!r},',
+            ])
+
+            op_ref_idents = [r.ident for r in x.refs if r.kind == 'op']
+            if op_ref_idents:
+                lines.extend([
+                    f'    op_ref_idents=(',
+                    *[
+                        f'        {r!r},'
+                        for r in sorted(op_ref_idents)
+                    ],
+                    f'    ),',
+                ])
+            else:
+                lines.append(
+                    '    op_ref_idents=(),',
+                )
+
+            lines.extend([
+                f'    cls_names=(',
+                *[
+                    f'        {cn!r},'
+                    for cn in sorted([
+                        (y.mod_name, y.cls_qualname)
+                        for y in grp
+                    ])
+                ],
+                f'    ),',
+            ])
+
+            lines.append(
+                ')',
+            )
+
+            lines.extend([
+                f'def {fn_name}():',
+            ])
+
+            lines.extend([
+                f'    {l}' if l.strip() else ''
+                for l in x.fn_lines
+            ])
+
+            lines.extend([
+                '',
+                f'    return {x.fn_name}',
+            ])
+
+        lines.append('')
+
+        with open(gen_file_path, 'w') as f:  # noqa
+            f.write('\n'.join(lines))
+
+    async def run(
             self,
             root_dirs: ta.Iterable[str],
+            *,
+            concurrency: int | None = 4,
     ) -> None:
         check.not_isinstance(root_dirs, str)
 
-        config_trie = self.build_config_trie(root_dirs)
+        cfg_pkgs = self.find_configured_packages(root_dirs)
+
+        cfg_pkg_trie = col.Trie([  # noqa
+            (cfg_pkg.name.split('.'), cfg_pkg)
+            for cfg_pkg in cfg_pkgs
+        ])
 
-        for pkg_parts, pkg_config in config_trie.iteritems(sort_children=True):
-            self.run_package_config('.'.join(pkg_parts), pkg_config)
+        await au.wait_maybe_concurrent([
+            self.process_configured_package(cfg_pkg)
+            for cfg_pkg in cfg_pkgs
+        ], concurrency)

omdev/dataclasses/dumping.py

@@ -0,0 +1,200 @@
+# ruff: noqa: UP006 UP007 UP037 UP045
+# @omlish-lite
+# @omlish-amalg _dumping.py
+import dataclasses as dc
+import os.path
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.json import json_dumps_pretty
+from omlish.lite.marshal import marshal_obj
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DumpedDataclassCodegen:
+    mod_name: str
+
+    cls_module: str
+    cls_qualname: str
+
+    plan_repr: str
+
+    fn_name: str
+    fn_params: ta.Sequence[str]
+
+    hdr_lines: ta.Sequence[str]
+    fn_lines: ta.Sequence[str]
+
+    @dc.dataclass(frozen=True)
+    class Ref:
+        kind: ta.Literal['op', 'global']
+        ident: str
+
+    refs: ta.Sequence[Ref]
+
+
+@dc.dataclass(frozen=True)
+class DataclassCodegenDumperOutput:
+    init_file_path: str
+    out_file_path: str
+
+    processed_modules: ta.Sequence[str]
+    import_errors: ta.Mapping[str, str]
+
+    dumped: ta.Sequence[DumpedDataclassCodegen]
+
+
+##
+
+
+class _DataclassCodegenDumper:
+    def __call__(
+            self,
+            *,
+            init_file_path: str,
+            out_file_path: str,
+    ) -> None:
+        from omlish.dataclasses.impl.configs import PACKAGE_CONFIG_CACHE  # noqa
+        from omlish.dataclasses.impl.generation.compilation import OpCompiler  # noqa
+        from omlish.dataclasses.impl.generation.globals import FnGlobal  # noqa
+        from omlish.dataclasses.impl.generation.ops import OpRef  # noqa
+        from omlish.dataclasses.impl.generation.processor import Codegen  # noqa
+        from omlish.dataclasses.impl.generation.processor import GeneratorProcessor  # noqa
+        from omlish.dataclasses.impl.processing.base import ProcessingContext  # noqa
+        from omlish.dataclasses.impl.processing.driving import processing_options_context  # noqa
+
+        cur_module: ta.Optional[str] = None
+
+        dumped: ta.List[DumpedDataclassCodegen] = []
+
+        def callback(
+                ctx: ProcessingContext,
+                prepared: GeneratorProcessor.Prepared,
+                comp: OpCompiler.CompileResult,
+        ) -> None:
+            d_refs: ta.List[DumpedDataclassCodegen.Ref] = []
+            for ref in comp.refs:
+                if isinstance(ref, OpRef):
+                    d_refs.append(DumpedDataclassCodegen.Ref(
+                        kind='op',
+                        ident=ref.ident(),
+                    ))
+                elif isinstance(ref, FnGlobal):
+                    d_refs.append(DumpedDataclassCodegen.Ref(
+                        kind='global',
+                        ident=ref.ident,
+                    ))
+                else:
+                    raise TypeError(ref)
+
+            dumped.append(DumpedDataclassCodegen(
+                mod_name=check.not_none(cur_module),
+
+                cls_module=ctx.cls.__module__,
+                cls_qualname=ctx.cls.__qualname__,
+
+                plan_repr=repr(prepared.plans),
+
+                fn_name=comp.fn_name,
+                fn_params=comp.fn_params,
+
+                hdr_lines=comp.hdr_lines,
+                fn_lines=comp.fn_lines,
+
+                refs=d_refs,
+            ))
+
+        #
+
+        processed_modules: ta.List[str] = []
+
+        import_errors: ta.Dict[str, str] = {}
+
+        def process_module(spec: str) -> None:
+            nonlocal cur_module
+            check.none(cur_module)
+            cur_module = spec
+
+            try:
+                processed_modules.append(spec)
+
+                try:
+                    __import__(spec)
+                except Exception as e:  # noqa
+                    import_errors[spec] = repr(e)
+
+            finally:
+                cur_module = None
+
+        def process_dir(dir_path: str) -> None:
+            spec = '.'.join(dir_path.split(os.path.sep))
+
+            process_module(spec)
+
+            pkg_cfg = PACKAGE_CONFIG_CACHE.get(spec)
+            if pkg_cfg is not None and not pkg_cfg.cfg.codegen:
+                return
+
+            #
+
+            dns: ta.List[str] = []
+            fns: ta.List[str] = []
+            for n in os.listdir(dir_path):
+                np = os.path.join(dir_path, n)
+                if os.path.isdir(np):
+                    dns.append(n)
+                elif os.path.isfile(np):
+                    fns.append(n)
+
+            for fn in sorted(fns):
+                if not fn.endswith('.py') or fn in ('conftest.py', '_dataclasses.py'):
+                    continue
+
+                fp = os.path.join(dir_path, fn)
+
+                fpp = fp.split(os.path.sep)
+                check.state(fpp[-1].endswith('.py'))
+                fpp[-1] = fpp[-1][:-3]
+                if fpp[-1] == '__init__':
+                    fpp.pop()
+                spec = '.'.join(fpp)
+
+                process_module(spec)
+
+            for dn in sorted(dns):
+                if dn == 'tests':
+                    continue
+
+                dp = os.path.join(dir_path, dn)
+
+                if not os.path.isfile(os.path.join(dp, '__init__.py')):
+                    continue
+
+                process_dir(dp)
+
+        #
+
+        with processing_options_context(Codegen(
+                style='aot',
+                force=True,
+                callback=callback,
+        )):
+            process_dir(os.path.dirname(init_file_path))
+
+        #
+
+        output = DataclassCodegenDumperOutput(
+            init_file_path=init_file_path,
+            out_file_path=out_file_path,
+
+            processed_modules=processed_modules,
+            import_errors=import_errors,
+
+            dumped=dumped,
+        )
+
+        with open(out_file_path, 'w') as f:
+            f.write(json_dumps_pretty(marshal_obj(output)))

omdev/interp/cli.py

@@ -17,7 +17,7 @@ from omlish.lite.check import check
 from omlish.lite.inject import Injector
 from omlish.lite.inject import inj
 from omlish.lite.runtime import check_lite_runtime_version
-from omlish.logs.standard import configure_standard_logging
+from omlish.logs.std.standard import configure_standard_logging
 
 from .inject import bind_interp
 from .resolvers import InterpResolver

omdev/interp/types.py

@@ -85,9 +85,10 @@ class InterpSpecifier:
     def parse(cls, s: str) -> 'InterpSpecifier':
         s, o = InterpOpts.parse_suffix(s)
         if not any(s.startswith(o) for o in Specifier.OPERATORS):
-            s = '~=' + s
             if s.count('.') < 2:
-                s += '.0'
+                s = '~=' + s + '.0'
+            else:
+                s = '==' + s
         return cls(
             specifier=Specifier(s),
             opts=o,