omdev 0.0.0.dev99__tar.gz → 0.0.0.dev101__tar.gz

{omdev-0.0.0.dev99/omdev.egg-info → omdev-0.0.0.dev101}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev99
+Version: 0.0.0.dev101
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev99
+Requires-Dist: omlish==0.0.0.dev101
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/.manifests.json

@@ -71,18 +71,6 @@
       }
     }
   },
-  {
-    "module": ".findmagic",
-    "attr": "_CLI_MODULE",
-    "file": "omdev/findmagic.py",
-    "line": 65,
-    "value": {
-      "$.cli.types.CliModule": {
-        "cmd_name": "py/findmagic",
-        "mod_name": "omdev.findmagic"
-      }
-    }
-  },
   {
     "module": ".imgur",
     "attr": "_FOO_CLI_MODULE",
@@ -107,6 +95,18 @@
       }
     }
   },
+  {
+    "module": ".magic.find",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/magic/find.py",
+    "line": 230,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": "py/findmagic",
+        "mod_name": "omdev.magic.find"
+      }
+    }
+  },
   {
     "module": ".precheck.__main__",
     "attr": "_CLI_MODULE",

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/amalg/amalg.py

@@ -41,7 +41,7 @@ from omlish import lang
 from omlish import logs
 from omlish.lite.runtime import REQUIRED_PYTHON_VERSION
 
-from .. import findmagic
+from .. import magic
 from .. import tokens as tks
 
 
@@ -94,11 +94,14 @@ def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
 
 
 STRIPPED_HEADER_MAGICS = [
-    '# @omlish-lite',
-    '# @omlish-script',
+    '@omlish-lite',
+    '@omlish-script',
 ]
 
-STRIPPED_HEADER_PATS = [findmagic.compile_magic_pat(m) for m in STRIPPED_HEADER_MAGICS]
+STRIPPED_HEADER_PAT = magic.compile_magic_style_pat(
+    magic.PY_MAGIC_STYLE,
+    keys=STRIPPED_HEADER_MAGICS,
+)
 
 
 def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
@@ -107,7 +110,7 @@ def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
     out = []
     for l in hls:
         ls = tks.join_toks(l)
-        if not any(p.fullmatch(ls) for p in STRIPPED_HEADER_PATS):
+        if not STRIPPED_HEADER_PAT.fullmatch(ls):
             out.append(l)
     return out
 

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/cache/compute/cache.py

@@ -3,6 +3,7 @@ TODO:
  - filesystem OPTIONAL
  - also postgres + (s3?) blobstore
  - locking
+ - alt codecs? json?
 
 ==
 

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/cexts/cmake.py

@@ -37,7 +37,7 @@ from omlish import lang
 from omlish import logs
 
 from .. import cmake
-from .. import findmagic
+from .. import magic
 from ..cli import CliModule
 from .magic import CextMagic
 
@@ -137,10 +137,10 @@ class CmakeProjectGen:
                 out.append(e)
             elif os.path.isdir(e):
                 out.extend(
-                    findmagic.find_magic(
+                    magic.find_magic_files(
+                        CextMagic.STYLE,
                         [e],
-                        [CextMagic.MAGIC_COMMENT],
-                        CextMagic.FILE_EXTENSIONS,
+                        keys=[CextMagic.KEY],
                     ),
                 )
             else:

omdev-0.0.0.dev101/omdev/cexts/magic.py

@@ -0,0 +1,7 @@
+# @omlish-lite
+from ..magic.styles import C_MAGIC_STYLE
+
+
+class CextMagic:
+    KEY = '@omlish-cext'
+    STYLE = C_MAGIC_STYLE

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/cexts/scan.py

@@ -15,7 +15,7 @@ def scan_one(
         input_path: str,
         **kwargs: ta.Any,
 ) -> None:
-    if not any(input_path.endswith('.' + fx) for fx in CextMagic.FILE_EXTENSIONS):
+    if not any(input_path.endswith('.' + fx) for fx in CextMagic.STYLE.exts):
         return
 
     with open(input_path, 'rb') as f:
@@ -26,9 +26,9 @@ def scan_one(
     except UnicodeDecodeError:
         return
 
-    sls = [l for l in src.splitlines() if l.startswith(CextMagic.MAGIC_COMMENT)]
+    sls = [l for l in src.splitlines() if l.startswith('// ' + CextMagic.KEY)]  # FIXME
     for sl in sls:
-        sas = sl[len(CextMagic.MAGIC_COMMENT):].split()  # noqa
+        sas = sl[len(CextMagic.KEY) + 3:].split()  # noqa
 
         log.info('Found ext: %s', input_path)
 

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/clipboard/darwin.py

@@ -123,47 +123,47 @@ def cfstring_to_string(cf_string: CFStringRef) -> str:
 ##
 
 
-class OsxClipboardError(Exception):
+class DarwinClipboardError(Exception):
     pass
 
 
 @dc.dataclass(frozen=True)
-class StatusOsxClipboardError(OsxClipboardError):
+class StatusDarwinClipboardError(DarwinClipboardError):
     fn: str
     status: int
 
 
 @dc.dataclass(frozen=True)
-class OsxClipboardItem:
+class DarwinClipboardItem:
     type: str | None
     data: bytes | None
 
 
-def get_osx_clipboard_data(
+def get_darwin_clipboard_data(
         *,
         types: ta.Container[str | None] | None = None,
         strict: bool = False,
         types_only: bool = False,
-) -> list[OsxClipboardItem]:
-    lst: list[OsxClipboardItem] = []
+) -> list[DarwinClipboardItem]:
+    lst: list[DarwinClipboardItem] = []
 
     pasteboard = PasteboardRef()
     if status := aps.PasteboardCreate(kPasteboardClipboard, ct.byref(pasteboard)):
-        raise StatusOsxClipboardError('PasteboardCreate', status)
+        raise StatusDarwinClipboardError('PasteboardCreate', status)
 
     try:
         item_count = ct.c_ulong(0)
         if status := aps.PasteboardGetItemCount(pasteboard, ct.byref(item_count)):
-            raise StatusOsxClipboardError('PasteboardGetItemCount', status)
+            raise StatusDarwinClipboardError('PasteboardGetItemCount', status)
 
         for i in range(1, item_count.value + 1):
             item_id = PasteboardItemID()
             if status := aps.PasteboardGetItemIdentifier(pasteboard, i, ct.byref(item_id)):
-                raise StatusOsxClipboardError('PasteboardGetItemIdentifier', status)
+                raise StatusDarwinClipboardError('PasteboardGetItemIdentifier', status)
 
             data_types = CFArrayRef()
             if status := aps.PasteboardCopyItemFlavors(pasteboard, item_id, ct.byref(data_types)):
-                raise StatusOsxClipboardError('PasteboardCopyItemFlavors', status)
+                raise StatusDarwinClipboardError('PasteboardCopyItemFlavors', status)
             if not data_types:
                 continue
 
@@ -181,7 +181,7 @@ def get_osx_clipboard_data(
                         continue
 
                     if types_only:
-                        lst.append(OsxClipboardItem(
+                        lst.append(DarwinClipboardItem(
                             type=data_type_str,
                             data=None,
                         ))
@@ -191,7 +191,7 @@ def get_osx_clipboard_data(
                     if status := aps.PasteboardCopyItemFlavorData(pasteboard, item_id, data_type, ct.byref(data)):
                         if not strict:
                             continue
-                        raise StatusOsxClipboardError('PasteboardCopyItemFlavorData', status)
+                        raise StatusDarwinClipboardError('PasteboardCopyItemFlavorData', status)
                     if not data:
                         continue
 
@@ -200,7 +200,7 @@ def get_osx_clipboard_data(
                         data_ptr = cf.CFDataGetBytePtr(data)
                         data_bytes = ct.string_at(data_ptr, data_size)
 
-                        lst.append(OsxClipboardItem(
+                        lst.append(DarwinClipboardItem(
                             type=data_type_str,
                             data=data_bytes,
                         ))
@@ -221,7 +221,7 @@ def get_osx_clipboard_data(
 
 
 def _main() -> None:
-    for i in get_osx_clipboard_data():
+    for i in get_darwin_clipboard_data():
         print(i)
 
 

omdev-0.0.0.dev101/omdev/magic/__init__.py

@@ -0,0 +1,28 @@
+# @omlish-lite
+
+from .find import (  # noqa
+    chop_magic_block,
+    chop_magic_lines,
+    compile_magic_style_pat,
+    find_magic,
+    find_magic_files,
+    find_magic_py_modules,
+)
+
+from .magic import (  # noqa
+    Magic,
+)
+
+from .prepare import (  # noqa
+    MagicPrepareError,
+    json_magic_preparer,
+    py_compile_magic_preparer,
+    py_eval_magic_preparer,
+)
+
+from .styles import (  # noqa
+    C_MAGIC_STYLE,
+    MAGIC_KEY_PREFIX,
+    MagicStyle,
+    PY_MAGIC_STYLE,
+)

omdev-0.0.0.dev101/omdev/magic/find.py

@@ -0,0 +1,268 @@
+# ruff: noqa: UP006 UP007
+import functools
+import os.path
+import re
+import typing as ta
+
+from .magic import Magic
+from .prepare import MagicPrepareError
+from .prepare import py_compile_magic_preparer
+from .styles import C_MAGIC_STYLE
+from .styles import PY_MAGIC_STYLE
+from .styles import MagicStyle
+
+
+##
+
+
+def compile_magic_style_pat(
+        style: MagicStyle,
+        *,
+        keys: ta.Optional[ta.Iterable[str]] = None,
+) -> re.Pattern:
+    ps: ta.List[str] = []
+    if style.line_prefix is not None:
+        ps.append(style.line_prefix)
+    if style.block_prefix_suffix is not None:
+        ps.append(style.block_prefix_suffix[0])
+    if not ps:
+        raise Exception('No prefixes')
+
+    ms: ta.List[str] = []
+    if keys is not None:
+        if isinstance(keys, str):
+            raise TypeError(keys)
+        for k in keys:
+            if not k.startswith(style.key_prefix):
+                raise Exception(f'Key does not start with prefix: {k!r} {style.key_prefix!r}')
+            ms.extend([re.escape(p + k) for p in ps])
+    else:
+        ms = [re.escape(p + style.key_prefix) + r'\S*' for p in ps]
+    if not ms:
+        raise Exception('No matchers')
+
+    b = '|'.join(f'({m})' for m in ms)
+    s = '^(' + b + r')($|(\s.*))'
+    return re.compile(s)
+
+
+##
+
+
+def chop_magic_lines(
+        magic_key: str,
+        prefix: str,
+        lines: ta.Iterable[str],
+) -> ta.Optional[ta.List[str]]:
+    out: ta.List[str] = []
+    for i, line in enumerate(lines):
+        if not i:
+            if not line.startswith(prefix + magic_key):
+                return None
+            out.append(line[len(prefix) + len(magic_key) + 1:])
+        else:
+            if not line.startswith(prefix):
+                return None
+            out.append(line[len(prefix):])
+    return out
+
+
+def chop_magic_block(
+        magic_key: str,
+        prefix: str,
+        suffix: str,
+        lines: ta.Iterable[str],
+) -> ta.Optional[ta.List[str]]:
+    out: ta.List[str] = []
+    for i, line in enumerate(lines):
+        if not i:
+            if not line.startswith(prefix + magic_key):
+                return None
+            s = line[len(prefix) + len(magic_key) + 1:]
+            if s.rstrip().endswith(suffix):
+                out.append(s.rstrip()[:-len(suffix)])
+                break
+            out.append(s)
+        elif line.rstrip().endswith(suffix):
+            out.append(line.rstrip()[:-len(suffix)])
+            break
+        else:
+            out.append(line)
+    return out
+
+
+##
+
+
+def find_magic(
+        style: MagicStyle,
+        lines: ta.Sequence[str],
+        *,
+        file: ta.Optional[str] = None,
+        preparer: ta.Callable[[str], ta.Any] = py_compile_magic_preparer,
+) -> ta.List[Magic]:
+    out: ta.List[Magic] = []
+
+    start = 0
+    while start < len(lines):
+        start_line = lines[start]
+
+        chopper: ta.Callable[[ta.Iterable[str]], ta.Optional[ta.List[str]]]
+        if (
+                style.line_prefix is not None and
+                start_line.startswith(style.line_prefix + style.key_prefix)
+        ):
+            key = start_line[len(style.line_prefix):].split()[0]
+            chopper = functools.partial(
+                chop_magic_lines,
+                key,
+                style.line_prefix,
+            )
+
+        elif (
+                style.block_prefix_suffix is not None and
+                start_line.startswith(style.block_prefix_suffix[0] + style.key_prefix)
+        ):
+            key = start_line[len(style.block_prefix_suffix[0]):].split()[0]
+            chopper = functools.partial(
+                chop_magic_block,
+                key,
+                *style.block_prefix_suffix,
+            )
+
+        else:
+            start += 1
+            continue
+
+        end = start
+        magic: ta.Optional[Magic] = None
+        while end < len(lines):
+            block_lines = chopper(lines[start:end + 1])
+            if block_lines is None:
+                raise Exception(f'Failed to find magic block terminator : {file=} {start=} {end=}')
+
+            block_src = ''.join(block_lines)
+            if not block_src:
+                prepared = None
+            else:
+                try:
+                    prepared = preparer(block_src)
+                except MagicPrepareError:
+                    end += 1
+                    continue
+
+            magic = Magic(
+                key=key,
+                file=file,
+                start_line=start + 1,
+                end_line=end + 1,
+                body=block_src,
+                prepared=prepared,
+            )
+            break
+
+        if magic is None:
+            raise Exception(f'Failed to find magic block terminator : {file=} {start=} {end=}')
+
+        out.append(magic)
+        start = end + 1
+
+    return out
+
+
+##
+
+
+def find_magic_files(
+        style: MagicStyle,
+        roots: ta.Sequence[str],
+        *,
+        keys: ta.Optional[ta.Iterable[str]] = None,
+) -> ta.Iterator[str]:
+    if isinstance(roots, str):
+        raise TypeError(roots)
+
+    pat = compile_magic_style_pat(
+        style,
+        keys=keys,
+    )
+
+    for root in roots:
+        for dp, dns, fns in os.walk(root):  # noqa
+            for fn in fns:
+                if not any(fn.endswith(f'.{x}') for x in style.exts):
+                    continue
+
+                fp = os.path.join(dp, fn)
+                try:
+                    with open(fp) as f:
+                        src = f.read()
+                except UnicodeDecodeError:
+                    continue
+
+                if not any(pat.fullmatch(l) for l in src.splitlines()):
+                    continue
+
+                yield fp
+
+
+def find_magic_py_modules(
+        roots: ta.Sequence[str],
+        *,
+        style: MagicStyle = PY_MAGIC_STYLE,
+        **kwargs: ta.Any,
+) -> ta.Iterator[str]:
+    for fp in find_magic_files(style, roots, **kwargs):
+        dp = os.path.dirname(fp)
+        fn = os.path.basename(fp)
+
+        if fn == '__init__.py':
+            yield dp.replace(os.sep, '.')
+        elif fn.endswith('.py'):
+            yield fp[:-3].replace(os.sep, '.')
+        else:
+            yield fp
+
+
+##
+
+
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': 'py/findmagic',
+    'mod_name': __name__,
+}}
+
+
+if __name__ == '__main__':
+    def _main(argv=None) -> None:
+        import argparse
+
+        arg_parser = argparse.ArgumentParser()
+        arg_parser.add_argument('--style', '-s', default='py')
+        arg_parser.add_argument('--key', '-k', dest='keys', action='append')
+        arg_parser.add_argument('--modules', action='store_true')
+        arg_parser.add_argument('roots', nargs='*')
+        args = arg_parser.parse_args(argv)
+
+        style = {
+            'py': PY_MAGIC_STYLE,
+            'c': C_MAGIC_STYLE,
+        }[args.style]
+
+        kw: dict = dict(
+            roots=args.roots,
+            style=style,
+            keys=args.keys,
+        )
+
+        fn: ta.Callable
+        if args.modules:
+            fn = find_magic_py_modules
+        else:
+            fn = find_magic_files
+
+        for out in fn(**kw):
+            print(out)
+
+    _main()

omdev-0.0.0.dev101/omdev/magic/magic.py

@@ -0,0 +1,17 @@
+# ruff: noqa: UP007
+import dataclasses as dc
+import typing as ta
+
+
+@dc.dataclass(frozen=True)
+class Magic:
+    key: str
+
+    file: ta.Optional[str]
+
+    start_line: int
+    end_line: int
+
+    body: str
+
+    prepared: ta.Any

omdev-0.0.0.dev101/omdev/magic/prepare.py

@@ -0,0 +1,27 @@
+import json
+import typing as ta
+
+
+class MagicPrepareError(Exception):
+    pass
+
+
+def py_compile_magic_preparer(src: str) -> ta.Any:
+    try:
+        prepared = compile(f'({src})', '<magic>', 'eval')
+    except SyntaxError:
+        raise MagicPrepareError  # noqa
+    return prepared
+
+
+def py_eval_magic_preparer(src: str) -> ta.Any:
+    code = py_compile_magic_preparer(src)
+    return eval(code)  # noqa
+
+
+def json_magic_preparer(src: str) -> ta.Any:
+    try:
+        prepared = json.loads(src)
+    except json.JSONDecodeError:
+        raise MagicPrepareError  # noqa
+    return prepared

omdev-0.0.0.dev101/omdev/magic/styles.py

@@ -0,0 +1,33 @@
+# ruff: noqa: UP006 UP007
+import dataclasses as dc
+import typing as ta
+
+
+MAGIC_KEY_PREFIX = '@omlish-'
+
+
+@dc.dataclass(frozen=True)
+class MagicStyle:
+    name: str
+
+    exts: ta.FrozenSet[str] = frozenset()
+
+    key_prefix: str = MAGIC_KEY_PREFIX
+
+    line_prefix: ta.Optional[str] = None
+    block_prefix_suffix: ta.Optional[ta.Tuple[str, str]] = None
+
+
+PY_MAGIC_STYLE = MagicStyle(
+    name='py',
+    exts=frozenset(['py']),
+    line_prefix='# ',
+)
+
+
+C_MAGIC_STYLE = MagicStyle(
+    name='c',
+    exts=frozenset(['c', 'cc', 'cpp']),
+    line_prefix='// ',
+    block_prefix_suffix=('/* ', '*/'),
+)

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/manifests/build.py

@@ -32,7 +32,7 @@ from omlish.lite.json import json_dumps_pretty
 from omlish.lite.logs import configure_standard_logging
 from omlish.lite.logs import log
 
-from .. import findmagic
+from .. import magic
 from .load import ManifestLoader
 from .types import Manifest
 from .types import ManifestOrigin
@@ -44,7 +44,7 @@ T = ta.TypeVar('T')
 ##
 
 
-MANIFEST_MAGIC = '# @omlish-manifest'
+MANIFEST_MAGIC_KEY = '@omlish-manifest'
 
 _MANIFEST_GLOBAL_PAT = re.compile(r'^(?P<name>[A-Za-z_][A-Za-z0-9_]*)\s*=.*')
 
@@ -148,7 +148,7 @@ class ManifestBuilder:
         origins: ta.List[ManifestOrigin] = []
         lines = src.splitlines(keepends=True)
         for i, l in enumerate(lines):
-            if l.startswith(MANIFEST_MAGIC):
+            if l.startswith('# ' + MANIFEST_MAGIC_KEY):
                 if (m := _MANIFEST_GLOBAL_PAT.match(nl := lines[i + 1])) is None:
                     raise Exception(nl)
 
@@ -235,10 +235,10 @@ class ManifestBuilder:
         if not os.path.isdir(pkg_dir) or not os.path.isfile(os.path.join(pkg_dir, '__init__.py')):
             raise Exception(pkg_dir)
 
-        files = sorted(findmagic.find_magic(
+        files = sorted(magic.find_magic_files(
+            magic.PY_MAGIC_STYLE,
             [pkg_dir],
-            [MANIFEST_MAGIC],
-            ['py'],
+            keys=[MANIFEST_MAGIC_KEY],
         ))
         manifests: ta.List[Manifest] = list(itertools.chain.from_iterable(await asyncio.gather(*[
             self._spawn(

{omdev-0.0.0.dev99 → omdev-0.0.0.dev101}/omdev/precheck/lite.py

@@ -8,10 +8,10 @@ import subprocess
 import textwrap
 import typing as ta
 
-from omdev import findmagic
 from omlish import cached
 from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
 
+from .. import magic
 from .base import Precheck
 from .base import PrecheckContext
 
@@ -121,10 +121,10 @@ class LitePython8Precheck(Precheck['LitePython8Precheck.Config']):
         return vs
 
     async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
-        for fp in findmagic.find_magic(
+        for fp in magic.find_magic_files(
+                magic.PY_MAGIC_STYLE,
                 self._context.src_roots,
-                ['# @omlish-lite'],
-                ['py'],
+                keys=['@omlish-lite'],
         ):
             with open(fp) as f:  # noqa  # FIXME
                 src = f.read()