omdev 0.0.0.dev162__py3-none-any.whl → 0.0.0.dev164__py3-none-any.whl

omdev/amalg/srcfiles.py

@@ -0,0 +1,120 @@
+import dataclasses as dc
+import typing as ta
+
+import tokenize_rt as trt
+
+from omlish import collections as col
+from omlish import lang
+
+from .. import tokens as tks
+from .imports import Import
+from .imports import make_import
+from .manifests import comment_out_manifest_comment
+from .manifests import is_manifest_comment
+from .resources import build_resource_lines
+from .resources import is_root_level_resources_read
+from .strip import split_header_lines
+from .strip import strip_header_lines
+from .types import Tokens
+from .typing import Typing
+from .typing import is_root_level_if_type_checking_block
+from .typing import make_typing
+from .typing import skip_root_level_if_type_checking_block
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class SrcFile:
+    path: str
+
+    src: str = dc.field(repr=False)
+    tokens: Tokens = dc.field(repr=False)
+    lines: ta.Sequence[Tokens] = dc.field(repr=False)
+
+    header_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    imports: ta.Sequence[Import] = dc.field(repr=False)
+    typings: ta.Sequence[Typing] = dc.field(repr=False)
+    content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+
+    ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
+
+    has_binary_resources: bool = False
+
+
+def make_src_file(
+        path: str,
+        *,
+        mounts: ta.Mapping[str, str],
+) -> SrcFile:
+    with open(path) as f:
+        src = f.read().strip()
+
+    tokens = trt.src_to_tokens(src)
+    lines = tks.split_lines(tokens)
+
+    header_lines, cls = split_header_lines(lines)
+
+    header_lines = strip_header_lines(header_lines)
+    rnls, header_lines = col.partition(header_lines, lambda l: tks.join_toks(l).startswith('# ruff: noqa: '))
+
+    imps: list[Import] = []
+    tys: list[Typing] = []
+    ctls: list[Tokens] = []
+
+    has_binary_resources = False
+
+    i = 0
+    while i < len(cls):
+        line = cls[i]
+        i += 1
+
+        if (imp := make_import(
+                line,
+                src_path=path,
+                mounts=mounts,
+        )) is not None:
+            imps.append(imp)
+
+        elif (ty := make_typing(
+                line,
+                src_path=path,
+        )) is not None:
+            tys.append(ty)
+
+        elif is_manifest_comment(line):
+            out, i = comment_out_manifest_comment(line, cls, i)
+            ctls.extend(out)
+
+        elif is_root_level_if_type_checking_block(line):
+            i = skip_root_level_if_type_checking_block(cls, i)
+
+        elif (rsrc := is_root_level_resources_read(line)) is not None:
+            ctls.extend(build_resource_lines(
+                rsrc,
+                path,
+            ))
+
+            if rsrc.kind == 'binary':
+                has_binary_resources = True
+
+        else:
+            ctls.append(line)
+
+    return SrcFile(
+        path=path,
+
+        src=src,
+        tokens=tokens,
+        lines=lines,
+
+        header_lines=header_lines,
+        imports=imps,
+        typings=tys,
+        content_lines=ctls,
+
+        ruff_noqa=set(lang.flatten(tks.join_toks(l).strip().split()[3:] for l in rnls)),  # noqa
+
+        has_binary_resources=has_binary_resources,
+    )

omdev/amalg/strip.py

@@ -0,0 +1,70 @@
+import re
+import typing as ta
+
+from .. import magic
+from .. import tokens as tks
+from .types import Tokens
+
+
+##
+
+
+HEADER_NAMES = (*tks.WS_NAMES, 'COMMENT', 'STRING')
+
+
+def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[Tokens]]:
+    ws = []
+    nws = []
+    for line in (it := iter(lines)):
+        if line[0].name in HEADER_NAMES:
+            ws.append(line)
+        else:
+            nws.append(line)
+            nws.extend(it)
+            break
+    return ws, nws
+
+
+##
+
+
+IF_MAIN_PAT = re.compile(r'if\s+__name__\s+==\s+[\'"]__main__[\'"]\s*:')
+
+
+def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
+    out = []
+
+    for l in (it := iter(cls)):
+        if IF_MAIN_PAT.fullmatch(tks.join_toks(l).strip()):
+            for l in it:
+                if l[0].name not in ('INDENT', 'UNIMPORTANT_WS') and tks.join_toks(l).strip():
+                    break
+        else:
+            out.append(l)
+
+    return out
+
+
+##
+
+
+STRIPPED_HEADER_MAGICS = [
+    '@omlish-lite',
+    '@omlish-script',
+]
+
+STRIPPED_HEADER_PAT = magic.compile_magic_style_pat(
+    magic.PY_MAGIC_STYLE,
+    keys=STRIPPED_HEADER_MAGICS,
+)
+
+
+def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
+    if hls and tks.join_toks(hls[0]).startswith('#!'):
+        hls = hls[1:]
+    out = []
+    for l in hls:
+        ls = tks.join_toks(l)
+        if not STRIPPED_HEADER_PAT.fullmatch(ls):
+            out.append(l)
+    return out

omdev/amalg/types.py

@@ -0,0 +1,6 @@
+import typing as ta
+
+from .. import tokens as tks
+
+
+Tokens: ta.TypeAlias = tks.Tokens

omdev/amalg/typing.py

@@ -0,0 +1,106 @@
+import dataclasses as dc
+
+from .. import tokens as tks
+from .types import Tokens
+
+
+##
+
+
+TYPE_ALIAS_COMMENT = '# ta.TypeAlias'
+NOQA_TYPE_ALIAS_COMMENT = TYPE_ALIAS_COMMENT + '  # noqa'
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Typing:
+    src: str
+
+    src_path: str
+    line: int
+
+    toks: Tokens = dc.field(repr=False)
+
+
+def _is_typing(
+        lts: Tokens,
+        *,
+        exclude_newtypes: bool = False,
+) -> bool:
+    es = tks.join_toks(lts).strip()
+    if any(es.endswith(sfx) for sfx in (TYPE_ALIAS_COMMENT, NOQA_TYPE_ALIAS_COMMENT)):
+        return True
+
+    wts = list(tks.ignore_ws(lts, keep=['INDENT', 'UNINDENT']))
+    if not tks.match_toks(wts, [
+        ('NAME', None),
+        ('OP', '='),
+        ('NAME', 'ta'),
+        ('OP', '.'),
+        (None, None),
+    ]):
+        return False
+
+    if exclude_newtypes:
+        if wts[4].name == 'NAME' and wts[4].src == 'NewType':
+            return False
+
+    return True
+
+
+def make_typing(
+        lts: Tokens,
+        *,
+        src_path: str,
+) -> Typing | None:
+    if not lts or lts[0].name == 'UNIMPORTANT_WS':
+        return None
+
+    if not _is_typing(lts, exclude_newtypes=True):
+        return None
+
+    ft = next(iter(tks.ignore_ws(lts)))
+    return Typing(
+        src=tks.join_toks(lts),
+
+        src_path=src_path,
+        line=ft.line,
+
+        toks=lts,
+    )
+
+
+##
+
+
+def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
+    return tks.match_toks(tks.ignore_ws(lts, keep=['INDENT']), [
+        ('NAME', 'if'),
+        ('NAME', 'ta'),
+        ('OP', '.'),
+        ('NAME', 'TYPE_CHECKING'),
+        ('OP', ':'),
+    ])
+
+
+def skip_root_level_if_type_checking_block(
+        cls: list[Tokens],
+        i: int,
+) -> int:
+    def skip_block():
+        nonlocal i
+        while True:
+            nl = cls[i]
+            if nl and nl[0].name != 'INDENT':
+                return nl
+            i += 1
+
+    nl = skip_block()
+
+    if tks.match_toks(nl, [
+        ('DEDENT', None),
+        ('NAME', 'else'),
+    ]):
+        i += 1
+        skip_block()
+
+    return i

omdev/manifests/build.py

@@ -28,7 +28,7 @@ import typing as ta
 from omlish.lite.cached import cached_nullary
 from omlish.lite.json import json_dumps_pretty
 from omlish.lite.logs import log
-from omlish.manifests.load import ManifestLoader
+from omlish.manifests.load import MANIFEST_LOADER
 from omlish.manifests.types import Manifest
 from omlish.manifests.types import ManifestOrigin
 
@@ -271,11 +271,10 @@ def check_package_manifests(
     with open(manifests_file) as f:
         manifests_json = json.load(f)
 
-    ldr = ManifestLoader()
     for entry in manifests_json:
         manifest = Manifest(**entry)
         [(key, value_dct)] = manifest.value.items()
         if key.startswith('$.'):
             key = f'${name}{key[1:]}'
-        cls = ldr.load_cls(key)
+        cls = MANIFEST_LOADER.load_cls(key)
         value = cls(**value_dct)  # noqa

omdev/precheck/manifests.py

@@ -1,7 +1,7 @@
 import dataclasses as dc
 import typing as ta
 
-from omlish.manifests.load import ManifestLoader
+from omlish.manifests.load import MANIFEST_LOADER
 
 from .base import Precheck
 from .base import PrecheckContext
@@ -20,8 +20,7 @@ class ManifestsPrecheck(Precheck['ManifestsPrecheck.Config']):
 
     async def run(self) -> ta.AsyncGenerator[Precheck.Violation, None]:
         for src_root in self._context.src_roots:
-            ml = ManifestLoader()
             try:
-                ml.load(src_root)
+                MANIFEST_LOADER.load(src_root)
             except Exception as e:  # noqa
                 yield Precheck.Violation(self, f'Error loading manifest for {src_root}: {e!r}')

{omdev-0.0.0.dev162.dist-info → omdev-0.0.0.dev164.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev162
+Version: 0.0.0.dev164
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev162
+Requires-Dist: omlish==0.0.0.dev164
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev162.dist-info → omdev-0.0.0.dev164.dist-info}/RECORD

@@ -12,8 +12,16 @@ omdev/secrets.py,sha256=bcquaBIDKqX4UIKOzUuKrX7nxVCenj67rRHIMIrd9bk,540
 omdev/tokens.py,sha256=zh2TCAfCbcq8ZnoVdQ824jrTiwNy3XJ_oCqlZpLpcCY,1574
 omdev/wheelfile.py,sha256=yfupGcGkbFlmzGzKU64k_vmOKpaKnUlDWxeGn2KdekU,10005
 omdev/amalg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-omdev/amalg/__main__.py,sha256=h94M-VqZ3AFBU2a8zOsjeKK7RF6uINhTHl6OiGbVMgw,163
-omdev/amalg/amalg.py,sha256=qJ-Kllp4mdmx28nt_JTlQnuO-o4XZnTTZP7Uh448b68,19333
+omdev/amalg/__main__.py,sha256=QKvvBlPJalD8Ta6ry7cxChH0lHFj78SKHMMHGUtpFYI,162
+omdev/amalg/gen.py,sha256=wp5CmwMUibcaIJrvwE_gQ1kInhWWgpvRntAI0ONKsQg,5995
+omdev/amalg/imports.py,sha256=KNyuu0zWW63gOSsy4cl1mPs23YVAlICbaF1yEPF147o,2072
+omdev/amalg/main.py,sha256=Ce8apkFfBinNyKVfNutH6ih0tHFc6Ta7RRVyDV5cIMs,4008
+omdev/amalg/manifests.py,sha256=sKnNKsS51zFFD8BIzRUVKtX50dY8pXOtYWi4WHZ_NTU,932
+omdev/amalg/resources.py,sha256=8tQ2yOnGAvPd_YValZZH6_pu9wCY8QeLVBuRUcyScDw,2722
+omdev/amalg/srcfiles.py,sha256=LskAq7kfpH_GuDoNOZF-sxEZiqljDN-VMXxfOfwNfks,3109
+omdev/amalg/strip.py,sha256=lsIkwCle66VjlEFpeQtQ2_sKhpn4HpYN_r4KP6v0DmQ,1439
+omdev/amalg/types.py,sha256=P0VKeWxXKOMQISqHcEFqSBzHQIbIly_nzi1GbMz6EZo,86
+omdev/amalg/typing.py,sha256=q90Mp5JdJaNMLTTwNHSGVx-z6rgfvSZNoV2KRj4dOmg,2049
 omdev/antlr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/antlr/consts.py,sha256=8pR6r0m0P3hAiyiAoJZ-nptd2GYbZ98mxwPL9cpaRuw,279
 omdev/antlr/gen.py,sha256=QnPyVWUrJYqHoOc3XsXA8fJdoiwAj6pyUwdG-DVrIeA,2953
@@ -89,7 +97,7 @@ omdev/magic/prepare.py,sha256=V5jYT2AeFmazzPwk9sNismSouLwFXEoik6FwKcWCNUY,589
 omdev/magic/styles.py,sha256=YQ-HgwfvFWPj-o_705E7A-yehEn1G1hRNLPWpeWCK0U,605
 omdev/manifests/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/manifests/__main__.py,sha256=JqyVDyV7_jo-NZ3wSs5clDU_xCMlxzJv-XFohoZWQ7E,174
-omdev/manifests/build.py,sha256=jQoNRSt9_4uh25xtkxzxx7WQ14TOJlHOjz5MGJCAlk4,8781
+omdev/manifests/build.py,sha256=yX0QM6c60aauiF-8oDxrXhkrhhwHcRw_qft_IIW6LD8,8767
 omdev/manifests/main.py,sha256=7zRlyE0BDPqITEbChlTBRGulAvG1nUZPHXrerNExriE,2126
 omdev/mypy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/mypy/debug.py,sha256=WcZw-3Z1njg_KFGqi3DB6RuqbBa3dLArJnjVCuY1Mn0,3003
@@ -105,7 +113,7 @@ omdev/precheck/base.py,sha256=a_lGoFM-QhL8u8XDUYFhb-feEyfPbP4j8lcmNO51sHY,732
 omdev/precheck/git.py,sha256=APC5Ln7x0zDrQiGPRWPsBcVJK3vWhbU-brqR5M63JQA,849
 omdev/precheck/lite.py,sha256=eQjEBigjKj-y8xbDe0VnPPQYReGmxGERykyHi2cpEuM,3878
 omdev/precheck/main.py,sha256=QL2en3Vn8nnuRTnPeipK9ovc2KrXtv1CkT2YGPqxCio,2884
-omdev/precheck/manifests.py,sha256=JNWxY3e5uFsJJSjjOeZZJE9mEkV6o7Y4I89D-UIXWyg,780
+omdev/precheck/manifests.py,sha256=ulwuYeZ0vnRsj8uTUbQGifoBNwI82MAsJuffs3rVIak,760
 omdev/precheck/scripts.py,sha256=Xw9kkQzlDd_2V9av9qlaNpNZG9jZdy3TTo7x60MeR2I,1273
 omdev/ptk/__init__.py,sha256=QIu7cMeCKgNiXvIt7pXTESToJLuRMN0Qsxns_Z7ci0k,641
 omdev/ptk/apps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -156,9 +164,9 @@ omdev/tools/json/rendering.py,sha256=jNShMfCpFR9-Kcn6cUFuOChXHjg71diuTC4x7Ofmz-o
 omdev/tools/pawk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/tools/pawk/__main__.py,sha256=VCqeRVnqT1RPEoIrqHFSu4PXVMg4YEgF4qCQm90-eRI,66
 omdev/tools/pawk/pawk.py,sha256=Eckymn22GfychCQcQi96BFqRo_LmiJ-EPhC8TTUJdB4,11446
-omdev-0.0.0.dev162.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev162.dist-info/METADATA,sha256=M8B2RvEyqAF2ZjuyLpk2s0jMqJsH1KnLLMM3qck1XXI,1760
-omdev-0.0.0.dev162.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-omdev-0.0.0.dev162.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
-omdev-0.0.0.dev162.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev162.dist-info/RECORD,,
+omdev-0.0.0.dev164.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev164.dist-info/METADATA,sha256=83Zwo5XiW3PNQLrwzB3Kdi4NKBsU1LXpTyZ2sd1HaR4,1760
+omdev-0.0.0.dev164.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+omdev-0.0.0.dev164.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
+omdev-0.0.0.dev164.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev164.dist-info/RECORD,,