omlish 0.0.0.dev447__py3-none-any.whl → 0.0.0.dev484__py3-none-any.whl

omlish/dataclasses/impl/configs.py

@@ -1,6 +1,14 @@
+"""
+TODO:
+ - better support anonymous / weakref'd / unlaoded modules / etc.
+  - anonymous=False on class spec?
+"""
 import dataclasses as dc
+import threading
 import typing as ta
 
+from ... import check
+
 
 ##
 
@@ -19,52 +27,104 @@ DEFAULT_PACKAGE_CONFIG = PackageConfig()
 ##
 
 
-def init_package(
-        init_globals: ta.MutableMapping[str, ta.Any],
-        *,
-        codegen: bool = False,
-) -> None:
-    if init_globals['__name__'] != init_globals['__package__']:
-        raise NameError('Must call dataclasses.init_package from __init__')
+@dc.dataclass(frozen=True)
+class NamedPackageConfig:
+    pkg: str | None
+    cfg: PackageConfig
+
+
+DEFAULT_NAMED_PACKAGE_CONFIG = NamedPackageConfig(None, DEFAULT_PACKAGE_CONFIG)
 
 
 ##
 
 
+@ta.final
 class PackageConfigCache:
     def __init__(self) -> None:
         super().__init__()
 
-        self._dct: dict[str, PackageConfig | None] = {}
-
-    @ta.overload
-    def get(self, pkg: str, default: PackageConfig) -> PackageConfig:
-        ...
-
-    @ta.overload
-    def get(self, pkg: str, default: PackageConfig | None = None) -> PackageConfig | None:
-        ...
-
-    def get(self, pkg, default=None):
-        # try:
-        #     return self._dct[pkg]
-        # except KeyError:
-        #     pass
-        #
-        # try:
-        #     s = importlib.resources.read_text(pkg, PACKAGE_CONFIG_FILE_NAME)
-        # except FileNotFoundError:
-        #     self._dct[pkg] = None
-        # else:
-        #     c = PackageConfig.loads(s)
-        #     self._dct[pkg] = c
-        #     return c
-        #
-        # if '.' not in pkg:
-        #     return default
-        #
-        # return self.get(pkg.rpartition('.')[0], default)
-        return None
+        self._lock = threading.Lock()
+        self._root = PackageConfigCache._Node('', None)
+        self._nodes: dict[str, PackageConfigCache._Node] = {}
+
+    @ta.final
+    class _Node:
+        def __init__(self, pkg: str, n_cfg: NamedPackageConfig | None) -> None:
+            self.pkg = pkg
+            self.n_cfg = n_cfg
+
+            self.children: dict[str, PackageConfigCache._Node] = {}
+
+        def __repr__(self) -> str:
+            return f'{self.__class__.__name__}(pkg={self.pkg!r}, n_cfg={self.n_cfg!r})'
+
+    def _navigate(self, *parts: str) -> _Node:
+        node = self._root
+        for i, p in enumerate(parts):
+            if (child := node.children.get(p)) is None:
+                child_pkg = '.'.join(parts[:i + 1])
+                check.not_in(child_pkg, self._nodes)
+                child = node.children[p] = self._nodes[child_pkg] = PackageConfigCache._Node(child_pkg, node.n_cfg)
+            node = child
+        return node
+
+    def put(self, pkg: str, cfg: PackageConfig) -> None:
+        n_cfg = NamedPackageConfig(pkg, cfg)
+        check.non_empty_str(pkg)
+        with self._lock:
+            check.not_in(pkg, self._nodes)
+            parts = pkg.split('.')
+            parent = self._navigate(*parts[:-1])
+            check.not_in(parts[-1], parent.children)
+            parent.children[parts[-1]] = self._nodes[pkg] = PackageConfigCache._Node(pkg, n_cfg)
+
+    def get(self, pkg: str) -> NamedPackageConfig | None:
+        if not pkg:
+            return None
+
+        try:
+            node = self._nodes[pkg]
+        except KeyError:
+            pass
+        else:
+            return node.n_cfg
+
+        # Flimsy - if no config anywhere for root package then don't cache. Want to support unlimited anonymous modules
+        # which may be unloaded without polluting cache forever.
+        parts = pkg.split('.')
+        if parts[0] not in self._root.children:
+            return None
+
+        with self._lock:
+            try:
+                node = self._nodes[pkg]
+            except KeyError:
+                pass
+            else:
+                return node.n_cfg
+
+            node = self._navigate(*pkg.split('.'))
+            return node.n_cfg
 
 
 PACKAGE_CONFIG_CACHE = PackageConfigCache()
+
+
+##
+
+
+def init_package(
+        init_globals: ta.MutableMapping[str, ta.Any],
+        *,
+        codegen: bool = False,
+) -> None:
+    pkg = check.non_empty_str(init_globals['__package__'])
+    if init_globals['__name__'] not in (pkg, '__main__'):
+        raise NameError('Must call dataclasses.init_package from __init__')
+
+    pkg_cfg = PackageConfig(
+        codegen=codegen,
+    )
+
+    PACKAGE_CONFIG_CACHE.put(pkg, pkg_cfg)

omlish/dataclasses/impl/generation/compilation.py

@@ -64,7 +64,6 @@ class OpCompiler:
                     f'import {i}'
                     for i in FN_GLOBAL_IMPORTS
                 ],
-                '\n',
             ]
 
         def globals_ns(self) -> ta.Mapping[str, ta.Any]:
@@ -84,11 +83,14 @@ class OpCompiler:
     def style(self) -> Style:
         return self._style
 
-    @dc.dataclass(frozen=True)
+    @dc.dataclass(frozen=True, kw_only=True)
     class CompileResult:
         fn_name: str
-        params: ta.Sequence[str]
-        src: str
+        fn_params: ta.Sequence[str]
+
+        hdr_lines: ta.Sequence[str]
+        fn_lines: ta.Sequence[str]
+
         refs: frozenset[Ref]
 
     @dc.dataclass(frozen=True)
@@ -137,7 +139,10 @@ class OpCompiler:
     ) -> CompileResult:
         body_lines: list[str] = []
 
-        for op in ops:
+        for i, op in enumerate(ops):
+            if i:
+                body_lines.append('')
+
             if isinstance(op, SetAttrOp):
                 if isinstance(v := op.value, OpRef):
                     vs = v.ident()
@@ -202,8 +207,6 @@ class OpCompiler:
             else:
                 raise TypeError(op)
 
-            body_lines.append('')
-
         #
 
         refs = frozenset.union(*[get_op_refs(o) for o in ops])
@@ -226,14 +229,12 @@ class OpCompiler:
                 src=f'{k.ident}={v.src}' if not v.src.startswith('.') else k.ident,
                 noqa=k.ident != k.ident.lower() or not v.src.startswith('.'),
             )
-            for k, v in FN_GLOBALS.items()
+            for k, v in sorted(FN_GLOBALS.items(), key=lambda t: t[0])
         ])
 
-        lines: list[str] = []
-
-        lines.extend(self._style.header_lines())
+        #
 
-        lines.extend([
+        fn_lines = [
             f'def {fn_name}(',
             f'    *,',
             *[
@@ -245,15 +246,16 @@ class OpCompiler:
                 f'    {l}'
                 for l in body_lines
             ],
-        ])
+        ]
 
         #
 
-        src = '\n'.join(lines)
-
         return self.CompileResult(
-            fn_name,
-            [p.name for p in params],
-            src,
-            refs,
+            fn_name=fn_name,
+            fn_params=[p.name for p in params],
+
+            hdr_lines=self._style.header_lines(),
+            fn_lines=fn_lines,
+
+            refs=refs,
         )

omlish/dataclasses/impl/generation/globals.py

@@ -12,6 +12,7 @@ from .idents import IDENT_PREFIX
 ##
 
 
+@ta.final
 class FnGlobal(ta.NamedTuple):
     ident: str
 

omlish/dataclasses/impl/generation/ops.py

@@ -16,6 +16,7 @@ class _OpRef(ta.NamedTuple, ta.Generic[T]):
     name: str
 
 
+@ta.final
 class OpRef(_OpRef[T]):
     def __repr__(self) -> str:
         return f'OpRef(name={self.name!r})'

omlish/dataclasses/impl/generation/processor.py

@@ -10,6 +10,7 @@ import typing as ta
 
 from .... import check
 from .... import lang
+from ....logs import all as logs
 from ..processing.base import ProcessingContext
 from ..processing.base import ProcessingOption
 from ..processing.base import Processor
@@ -21,7 +22,6 @@ from .execution import OpExecutor
 from .globals import FN_GLOBALS
 from .globals import FnGlobal
 from .idents import CLS_IDENT
-from .mangling import IDENT_MANGLER
 from .ops import Op
 from .ops import OpRef
 from .ops import OpRefMap
@@ -30,6 +30,9 @@ from .registry import all_generator_types
 from .registry import generator_type_for_plan_type
 
 
+log = logs.get_module_logger(globals())
+
+
 ##
 
 
@@ -38,9 +41,10 @@ class PlanOnly(ProcessingOption):
     b: bool
 
 
-@dc.dataclass(frozen=True)
-class Verbose(ProcessingOption):
-    b: bool
+@dc.dataclass(frozen=True, kw_only=True)
+class Verbosity(ProcessingOption):
+    warn: bool = False
+    debug: bool = False
 
 
 class CompileCallback(ta.Protocol):
@@ -53,13 +57,21 @@ class CompileCallback(ta.Protocol):
         ...
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass(frozen=True, kw_only=True)
 class Codegen(ProcessingOption):
-    callback: CompileCallback
+    style: ta.Literal['jit', 'aot'] = 'jit'
+
+    force: bool = False
+    callback: CompileCallback | None = None
+
+
+##
 
 
 @register_processor_type(priority=ProcessorPriority.GENERATION)
 class GeneratorProcessor(Processor):
+    PROCESS_FN_NAME: ta.ClassVar[str] = '_process_dataclass'
+
     class Mode(lang.Abstract):
         @abc.abstractmethod
         def _process(self, gp: 'GeneratorProcessor', cls: type) -> None:
@@ -85,36 +97,52 @@ class GeneratorProcessor(Processor):
 
             self._codegen = codegen
 
+        @classmethod
+        def build_standard_kwargs(cls, dc_cls: type) -> dict[str, ta.Any]:
+            kw: dict = {CLS_IDENT: dc_cls}
+            kw.update({
+                k.ident: v.value
+                for k, v in FN_GLOBALS.items()
+            })
+            return kw
+
         def _process(self, gp: 'GeneratorProcessor', cls: type) -> None:
-            compiler = OpCompiler(
-                OpCompiler.AotStyle(),
-                # OpCompiler.JitStyle(),
-            )
+            style: OpCompiler.Style = {
+                'jit': OpCompiler.JitStyle,
+                'aot': OpCompiler.AotStyle,
+            }[self._codegen.style]()
 
-            fn_name = '_process_dataclass__' + IDENT_MANGLER.mangle(cls.__qualname__)
+            compiler = OpCompiler(style)
 
             comp = compiler.compile(
-                fn_name,
+                GeneratorProcessor.PROCESS_FN_NAME,
                 gp.ops(),
             )
 
-            if (vo := gp._ctx.option(Verbose)) is not None and vo.b:  # noqa
+            comp_src = '\n'.join([
+                *comp.hdr_lines,
+                *(['', ''] if comp.hdr_lines else []),
+                *comp.fn_lines,
+            ])
+
+            if (vo := gp._ctx.option(Verbosity)) is not None and vo.debug:  # noqa
                 print(gp.prepare().plans.render(), file=sys.stderr)
                 print(file=sys.stderr)
-                print(comp.src, file=sys.stderr)
+                print(comp_src, file=sys.stderr)
                 print(file=sys.stderr)
 
             ns: dict = {}
             ns.update(compiler.style.globals_ns())  # noqa
 
-            exec(comp.src, ns)
-            o_fn = ns[comp.fn_name]
+            exec(comp_src, ns)
+            o_fn = ns[GeneratorProcessor.PROCESS_FN_NAME]
 
             if cls.__module__ in sys.modules:
                 gl = sys.modules[cls.__module__].__dict__
             else:
                 gl = {}
 
+            # TODO: comment why lol
             fn = lang.new_function(**{
                 **lang.new_function_kwargs(o_fn),
                 **dict(
@@ -122,12 +150,8 @@ class GeneratorProcessor(Processor):
                 ),
             })
 
-            kw: dict = {CLS_IDENT: cls}
-            kw.update({
-                k.ident: v.value
-                for k, v in FN_GLOBALS.items()
-                # if v.src.startswith('.')
-            })
+            kw = self.build_standard_kwargs(cls)
+
             orm = gp.prepare().ref_map
             for r in comp.refs:
                 if isinstance(r, OpRef):
@@ -140,7 +164,7 @@ class GeneratorProcessor(Processor):
             fn(**kw)
 
             if (cg := self._codegen) is not None and (cb := cg.callback) is not None:
-                cb(
+                cb(  # noqa
                     gp._ctx,  # noqa
                     gp.prepare(),
                     comp,
@@ -194,13 +218,70 @@ class GeneratorProcessor(Processor):
 
     #
 
+    def _process_from_codegen(self, cls: type) -> bool:
+        cg_pkg = check.not_none(self._ctx.pkg_cfg.pkg)
+        cg_mod_spec = f'{cg_pkg}._dataclasses'
+
+        try:
+            __import__(cg_mod_spec)
+        except ImportError:
+            if (vo := self._ctx.option(Verbosity)) is not None and vo.warn:  # noqa
+                log.warning(lambda: f'Codegen module missing for {cls.__module__}.{cls.__qualname__} at {cg_mod_spec}')
+            return False
+
+        cg_mod = sys.modules[cg_mod_spec]
+        cg_fn_reg = cg_mod.REGISTRY
+
+        #
+
+        prep = self.prepare()
+        prep_plan_repr = repr(prep.plans)
+
+        #
+
+        try:
+            cg_reg_item = cg_fn_reg[prep_plan_repr]
+        except KeyError:
+            if (vo := self._ctx.option(Verbosity)) is not None and vo.warn:  # noqa
+                log.warning(lambda: f'Codegen missing for {cls.__module__}.{cls.__qualname__} in {cg_mod_spec}')
+            return False
+
+        cg_kw, cg_fn = cg_reg_item
+
+        #
+
+        ref_map = {
+            ref.ident(): v
+            for ref, v in prep.ref_map.items()
+        }
+
+        #
+
+        fn_kw = {
+            **GeneratorProcessor.CompilerMode.build_standard_kwargs(cls),
+            **{k: ref_map[k] for k in cg_kw['op_ref_idents']},
+        }
+
+        fn = cg_fn()
+        fn(**fn_kw)
+
+        return True
+
+    #
+
     def process(self, cls: type) -> type:
         if (po := self._ctx.option(PlanOnly)) is not None and po.b:
             self.prepare()
             return cls
 
+        cg = self._ctx.option(Codegen)
+
+        if not (cg is not None and cg.force) and self._ctx.pkg_cfg.cfg.codegen:
+            if self._process_from_codegen(cls):
+                return cls
+
         mode: GeneratorProcessor.Mode
-        if (cg := self._ctx.option(Codegen)) is not None:  # noqa
+        if cg is not None:  # noqa
             mode = GeneratorProcessor.CompilerMode(codegen=cg)
         else:
             mode = GeneratorProcessor.ExecutorMode()

omlish/dataclasses/impl/processing/base.py

@@ -10,6 +10,8 @@ import typing as ta
 from .... import check
 from .... import lang
 from ...specs import ClassSpec
+from ..configs import DEFAULT_NAMED_PACKAGE_CONFIG
+from ..configs import NamedPackageConfig
 
 
 T = ta.TypeVar('T')
@@ -33,6 +35,7 @@ class ProcessingContext:
             cs: ClassSpec,
             item_factories: ta.Mapping[type, ProcessingContextItemFactory],
             *,
+            pkg_cfg: NamedPackageConfig = DEFAULT_NAMED_PACKAGE_CONFIG,
             options: ta.Sequence[ProcessingOption] | None = None,
     ) -> None:
         super().__init__()
@@ -40,6 +43,7 @@ class ProcessingContext:
         self._cls = cls
         self._cs = cs
         self._item_factories = item_factories
+        self._pkg_cfg = pkg_cfg
 
         options_dct: dict = {}
         for o in options or ():
@@ -57,6 +61,10 @@ class ProcessingContext:
     def cs(self) -> ClassSpec:
         return self._cs
 
+    @property
+    def pkg_cfg(self) -> NamedPackageConfig:
+        return self._pkg_cfg
+
     def __getitem__(self, ty: type[T]) -> T:
         try:
             return self._items[ty]

omlish/dataclasses/impl/processing/driving.py

@@ -5,7 +5,7 @@ import typing as ta
 from .... import lang
 from ...specs import ClassSpec
 from .. import concerns as _concerns  # noqa  # imported for registration
-from ..configs import DEFAULT_PACKAGE_CONFIG
+from ..configs import DEFAULT_NAMED_PACKAGE_CONFIG
 from ..configs import PACKAGE_CONFIG_CACHE
 from ..generation import processor as gp
 from .base import ProcessingContext
@@ -41,21 +41,20 @@ def drive_cls_processing(
         cs: ClassSpec,
         *,
         plan_only: bool = False,
-        verbose: bool = False,
+        warn: bool = False,
+        debug: bool = False,
 ) -> type:
     options: list[ProcessingOption] = list(_OPTIONS_CONTEXT_VAR.get())
     if plan_only:
         options.append(gp.PlanOnly(True))
-    if verbose:
-        options.append(gp.Verbose(True))
+    if warn or debug:
+        options.append(gp.Verbosity(warn=warn, debug=debug))
 
     #
 
-    pkg_config = DEFAULT_PACKAGE_CONFIG
     cls_mod = cls.__module__
-    if '.' in cls_mod:
-        cls_pkg = cls_mod.rpartition('.')[0]
-        pkg_cfg = PACKAGE_CONFIG_CACHE.get(cls_pkg, pkg_config)  # noqa
+    cls_pkg = cls_mod.rpartition('.')[0]
+    pkg_cfg = lang.coalesce(PACKAGE_CONFIG_CACHE.get(cls_pkg), DEFAULT_NAMED_PACKAGE_CONFIG)
 
     #
 
@@ -63,6 +62,7 @@ def drive_cls_processing(
         cls,
         cs,
         all_processing_context_item_factories(),
+        pkg_cfg=pkg_cfg,
         options=options,
     )
 

omlish/dataclasses/specs.py

@@ -216,6 +216,7 @@ class ClassSpec(_SpecBase, lang.Final):
 
     repr_id: bool = False
     terse_repr: bool = False
+    default_repr_fn: ReprFn | None = None
 
     allow_redundant_decorator: bool = False
 

omlish/dataclasses/tools/modifiers.py

@@ -3,6 +3,7 @@ import typing as ta
 
 from ... import check
 from ... import lang
+from ...lite.dataclasses import is_immediate_dataclass
 
 
 if ta.TYPE_CHECKING:
@@ -40,6 +41,8 @@ def update_fields(
     def inner(cls):
         if issubclass(cls, meta.DataMeta):
             raise TypeError('update_fields() cannot be used on DataMeta subclasses')
+        if is_immediate_dataclass(cls):
+            raise TypeError('update_fields() cannot be used on already processed dataclasses')
 
         if fields is None:
             for a, v in list(cls.__dict__.items()):
@@ -51,6 +54,8 @@ def update_fields(
                 try:
                     v = cls.__dict__[a]
                 except KeyError:
+                    if hasattr(cls, a):
+                        raise TypeError('update_fields() cannot be used on parent dataclass fields') from None
                     v = dc.field()
                 else:
                     if not isinstance(v, dc.Field):

omlish/diag/{lslocks.py → cmds/lslocks.py}

@@ -7,12 +7,12 @@ import dataclasses as dc
 import json
 import typing as ta
 
-from ..lite.check import check
-from ..lite.marshal import OBJ_MARSHALER_FIELD_KEY
-from ..lite.marshal import unmarshal_obj
-from ..subprocesses.run import SubprocessRun
-from ..subprocesses.run import SubprocessRunnable
-from ..subprocesses.run import SubprocessRunOutput
+from ...lite.check import check
+from ...lite.marshal import OBJ_MARSHALER_FIELD_KEY
+from ...lite.marshal import unmarshal_obj
+from ...subprocesses.run import SubprocessRun
+from ...subprocesses.run import SubprocessRunnable
+from ...subprocesses.run import SubprocessRunOutput
 
 
 ##

omlish/diag/{lsof.py → cmds/lsof.py}

@@ -7,12 +7,12 @@ import dataclasses as dc
 import enum
 import typing as ta
 
-from ..lite.check import check
-from ..lite.dataclasses import dataclass_repr_omit_falsey
-from ..lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
-from ..subprocesses.run import SubprocessRun
-from ..subprocesses.run import SubprocessRunnable
-from ..subprocesses.run import SubprocessRunOutput
+from ...lite.check import check
+from ...lite.dataclasses import dataclass_repr_omit_falsey
+from ...lite.marshal import OBJ_MARSHALER_OMIT_IF_NONE
+from ...subprocesses.run import SubprocessRun
+from ...subprocesses.run import SubprocessRunnable
+from ...subprocesses.run import SubprocessRunOutput
 
 
 ##

omlish/diag/{ps.py → cmds/ps.py}

@@ -4,12 +4,12 @@ import dataclasses as dc
 import os
 import typing as ta
 
-from ..lite.check import check
-from ..lite.timeouts import Timeout
-from ..subprocesses.run import SubprocessRun
-from ..subprocesses.run import SubprocessRunnable
-from ..subprocesses.run import SubprocessRunOutput
-from ..subprocesses.sync import subprocesses
+from ...lite.check import check
+from ...lite.timeouts import Timeout
+from ...subprocesses.run import SubprocessRun
+from ...subprocesses.run import SubprocessRunnable
+from ...subprocesses.run import SubprocessRunOutput
+from ...subprocesses.sync import subprocesses
 
 
 ##

omlish/diag/pycharm.py

@@ -191,9 +191,23 @@ def pycharm_remote_debugger_attach(prd: PycharmRemoteDebugger) -> None:
     else:
         pydevd_pycharm = _import_pydevd_pycharm(version=version)
 
+    import inspect
+    st_sig = inspect.signature(pydevd_pycharm.settrace)
+
+    kw: dict = {}
+    if 'stdoutToServer' in st_sig.parameters:
+        kw.update(
+            stdoutToServer=True,
+            stderrToServer=True,
+        )
+    else:
+        kw.update(
+            stdout_to_server=True,
+            stderr_to_server=True,
+        )
+
     pydevd_pycharm.settrace(
         host,
         port=prd.port,
-        stdoutToServer=True,
-        stderrToServer=True,
+        **kw,
     )