omdev 0.0.0.dev163__py3-none-any.whl → 0.0.0.dev165__py3-none-any.whl

omdev/amalg/srcfiles.py

@@ -0,0 +1,120 @@
+import dataclasses as dc
+import typing as ta
+
+import tokenize_rt as trt
+
+from omlish import collections as col
+from omlish import lang
+
+from .. import tokens as tks
+from .imports import Import
+from .imports import make_import
+from .manifests import comment_out_manifest_comment
+from .manifests import is_manifest_comment
+from .resources import build_resource_lines
+from .resources import is_root_level_resources_read
+from .strip import split_header_lines
+from .strip import strip_header_lines
+from .types import Tokens
+from .typing import Typing
+from .typing import is_root_level_if_type_checking_block
+from .typing import make_typing
+from .typing import skip_root_level_if_type_checking_block
+
+
+##
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class SrcFile:
+    path: str
+
+    src: str = dc.field(repr=False)
+    tokens: Tokens = dc.field(repr=False)
+    lines: ta.Sequence[Tokens] = dc.field(repr=False)
+
+    header_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    imports: ta.Sequence[Import] = dc.field(repr=False)
+    typings: ta.Sequence[Typing] = dc.field(repr=False)
+    content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+
+    ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
+
+    has_binary_resources: bool = False
+
+
+def make_src_file(
+        path: str,
+        *,
+        mounts: ta.Mapping[str, str],
+) -> SrcFile:
+    with open(path) as f:
+        src = f.read().strip()
+
+    tokens = trt.src_to_tokens(src)
+    lines = tks.split_lines(tokens)
+
+    header_lines, cls = split_header_lines(lines)
+
+    header_lines = strip_header_lines(header_lines)
+    rnls, header_lines = col.partition(header_lines, lambda l: tks.join_toks(l).startswith('# ruff: noqa: '))
+
+    imps: list[Import] = []
+    tys: list[Typing] = []
+    ctls: list[Tokens] = []
+
+    has_binary_resources = False
+
+    i = 0
+    while i < len(cls):
+        line = cls[i]
+        i += 1
+
+        if (imp := make_import(
+                line,
+                src_path=path,
+                mounts=mounts,
+        )) is not None:
+            imps.append(imp)
+
+        elif (ty := make_typing(
+                line,
+                src_path=path,
+        )) is not None:
+            tys.append(ty)
+
+        elif is_manifest_comment(line):
+            out, i = comment_out_manifest_comment(line, cls, i)
+            ctls.extend(out)
+
+        elif is_root_level_if_type_checking_block(line):
+            i = skip_root_level_if_type_checking_block(cls, i)
+
+        elif (rsrc := is_root_level_resources_read(line)) is not None:
+            ctls.extend(build_resource_lines(
+                rsrc,
+                path,
+            ))
+
+            if rsrc.kind == 'binary':
+                has_binary_resources = True
+
+        else:
+            ctls.append(line)
+
+    return SrcFile(
+        path=path,
+
+        src=src,
+        tokens=tokens,
+        lines=lines,
+
+        header_lines=header_lines,
+        imports=imps,
+        typings=tys,
+        content_lines=ctls,
+
+        ruff_noqa=set(lang.flatten(tks.join_toks(l).strip().split()[3:] for l in rnls)),  # noqa
+
+        has_binary_resources=has_binary_resources,
+    )

omdev/amalg/strip.py

@@ -0,0 +1,70 @@
+import re
+import typing as ta
+
+from .. import magic
+from .. import tokens as tks
+from .types import Tokens
+
+
+##
+
+
+HEADER_NAMES = (*tks.WS_NAMES, 'COMMENT', 'STRING')
+
+
+def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[Tokens]]:
+    ws = []
+    nws = []
+    for line in (it := iter(lines)):
+        if line[0].name in HEADER_NAMES:
+            ws.append(line)
+        else:
+            nws.append(line)
+            nws.extend(it)
+            break
+    return ws, nws
+
+
+##
+
+
+IF_MAIN_PAT = re.compile(r'if\s+__name__\s+==\s+[\'"]__main__[\'"]\s*:')
+
+
+def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
+    out = []
+
+    for l in (it := iter(cls)):
+        if IF_MAIN_PAT.fullmatch(tks.join_toks(l).strip()):
+            for l in it:
+                if l[0].name not in ('INDENT', 'UNIMPORTANT_WS') and tks.join_toks(l).strip():
+                    break
+        else:
+            out.append(l)
+
+    return out
+
+
+##
+
+
+STRIPPED_HEADER_MAGICS = [
+    '@omlish-lite',
+    '@omlish-script',
+]
+
+STRIPPED_HEADER_PAT = magic.compile_magic_style_pat(
+    magic.PY_MAGIC_STYLE,
+    keys=STRIPPED_HEADER_MAGICS,
+)
+
+
+def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
+    if hls and tks.join_toks(hls[0]).startswith('#!'):
+        hls = hls[1:]
+    out = []
+    for l in hls:
+        ls = tks.join_toks(l)
+        if not STRIPPED_HEADER_PAT.fullmatch(ls):
+            out.append(l)
+    return out

omdev/amalg/types.py

@@ -0,0 +1,6 @@
+import typing as ta
+
+from .. import tokens as tks
+
+
+Tokens: ta.TypeAlias = tks.Tokens

omdev/amalg/typing.py

@@ -0,0 +1,106 @@
+import dataclasses as dc
+
+from .. import tokens as tks
+from .types import Tokens
+
+
+##
+
+
+TYPE_ALIAS_COMMENT = '# ta.TypeAlias'
+NOQA_TYPE_ALIAS_COMMENT = TYPE_ALIAS_COMMENT + '  # noqa'
+
+
+@dc.dataclass(frozen=True, kw_only=True)
+class Typing:
+    src: str
+
+    src_path: str
+    line: int
+
+    toks: Tokens = dc.field(repr=False)
+
+
+def _is_typing(
+        lts: Tokens,
+        *,
+        exclude_newtypes: bool = False,
+) -> bool:
+    es = tks.join_toks(lts).strip()
+    if any(es.endswith(sfx) for sfx in (TYPE_ALIAS_COMMENT, NOQA_TYPE_ALIAS_COMMENT)):
+        return True
+
+    wts = list(tks.ignore_ws(lts, keep=['INDENT', 'UNINDENT']))
+    if not tks.match_toks(wts, [
+        ('NAME', None),
+        ('OP', '='),
+        ('NAME', 'ta'),
+        ('OP', '.'),
+        (None, None),
+    ]):
+        return False
+
+    if exclude_newtypes:
+        if wts[4].name == 'NAME' and wts[4].src == 'NewType':
+            return False
+
+    return True
+
+
+def make_typing(
+        lts: Tokens,
+        *,
+        src_path: str,
+) -> Typing | None:
+    if not lts or lts[0].name == 'UNIMPORTANT_WS':
+        return None
+
+    if not _is_typing(lts, exclude_newtypes=True):
+        return None
+
+    ft = next(iter(tks.ignore_ws(lts)))
+    return Typing(
+        src=tks.join_toks(lts),
+
+        src_path=src_path,
+        line=ft.line,
+
+        toks=lts,
+    )
+
+
+##
+
+
+def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
+    return tks.match_toks(tks.ignore_ws(lts, keep=['INDENT']), [
+        ('NAME', 'if'),
+        ('NAME', 'ta'),
+        ('OP', '.'),
+        ('NAME', 'TYPE_CHECKING'),
+        ('OP', ':'),
+    ])
+
+
+def skip_root_level_if_type_checking_block(
+        cls: list[Tokens],
+        i: int,
+) -> int:
+    def skip_block():
+        nonlocal i
+        while True:
+            nl = cls[i]
+            if nl and nl[0].name != 'INDENT':
+                return nl
+            i += 1
+
+    nl = skip_block()
+
+    if tks.match_toks(nl, [
+        ('DEDENT', None),
+        ('NAME', 'else'),
+    ]):
+        i += 1
+        skip_block()
+
+    return i

omdev/tools/cloc.py

@@ -71,15 +71,29 @@ def count_lines(file_path: str, language: str) -> FileLineCount:
     )
 
 
-def count_lines_in_directory(directory: str) -> ta.Mapping[str, FileLineCount]:
+def count_lines_in_directory(
+        directory: str,
+        *,
+        include: list[re.Pattern[str]] | None = None,
+        exclude: list[re.Pattern[str]] | None = None,
+) -> ta.Mapping[str, FileLineCount]:
     results: dict[str, FileLineCount] = {}
     for root, _, files in os.walk(directory):
         for file in files:
             ext = os.path.splitext(file)[1]
-            if ext in SUPPORTED_EXTENSIONS:
-                language = SUPPORTED_EXTENSIONS[ext]
-                file_path = os.path.join(root, file)
-                results[file_path] = count_lines(file_path, language)
+            if ext not in SUPPORTED_EXTENSIONS:
+                continue
+
+            file_path = os.path.join(root, file)
+
+            if include and not any(p.fullmatch(file_path) for p in include):
+                continue
+            if exclude and any(p.fullmatch(file_path) for p in exclude):
+                continue
+
+            language = SUPPORTED_EXTENSIONS[ext]
+            results[file_path] = count_lines(file_path, language)
+
     return results
 
 
@@ -132,14 +146,50 @@ def _main() -> None:
     import argparse
 
     parser = argparse.ArgumentParser(description='Count lines of code in source files.')
+
     parser.add_argument('directory', help='The directory to analyze.', nargs='+')
+
+    parser.add_argument('-i,', '--include', action='append')
+    parser.add_argument('-e,', '--exclude', action='append')
+
     args = parser.parse_args()
 
+    #
+
+    include: list[re.Pattern[str]] | None = None
+    if args.include:
+        include = [re.compile(p) for p in args.include]
+
+    exclude: list[re.Pattern[str]] | None = None
+    if args.exclude:
+        exclude = [re.compile(p) for p in args.exclude]
+
+    #
+
+    results_by_directory: dict[str, FileLineCount] = {}
     for directory in args.directory:
-        results = count_lines_in_directory(directory)
+        results = count_lines_in_directory(
+            directory,
+            include=include,
+            exclude=exclude,
+        )
+
+        if not results:
+            continue
+
         display_results(results)
         print()
 
+        results_by_directory[directory] = FileLineCount(
+            loc=sum(flc.loc for flc in results.values()),
+            blanks=sum(flc.blanks for flc in results.values()),
+            comments=sum(flc.comments for flc in results.values()),
+        )
+
+    if len(results_by_directory) > 1:
+        display_results(results_by_directory)
+        print()
+
 
 # @omlish-manifest
 _CLI_MODULE = CliModule('cloc', __name__)

{omdev-0.0.0.dev163.dist-info → omdev-0.0.0.dev165.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omdev
-Version: 0.0.0.dev163
+Version: 0.0.0.dev165
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev163
+Requires-Dist: omlish==0.0.0.dev165
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"

{omdev-0.0.0.dev163.dist-info → omdev-0.0.0.dev165.dist-info}/RECORD

@@ -1,4 +1,4 @@
-omdev/.manifests.json,sha256=Dm-Nd3V7Ry9d3-tO5OnUM7IYT2_ldruPEjxcSr4BA1U,8306
+omdev/.manifests.json,sha256=tF8O0ywJseQMbbhT2b67QAvjuAaHXLY1MyjeMrDObos,8306
 omdev/__about__.py,sha256=n5x-SO70OgbDQFzQ1d7sZDVMsnkQc4PxQZPFaIQFa0E,1281
 omdev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/bracepy.py,sha256=I8EdqtDvxzAi3I8TuMEW-RBfwXfqKbwp06CfOdj3L1o,2743
@@ -12,8 +12,16 @@ omdev/secrets.py,sha256=bcquaBIDKqX4UIKOzUuKrX7nxVCenj67rRHIMIrd9bk,540
 omdev/tokens.py,sha256=zh2TCAfCbcq8ZnoVdQ824jrTiwNy3XJ_oCqlZpLpcCY,1574
 omdev/wheelfile.py,sha256=yfupGcGkbFlmzGzKU64k_vmOKpaKnUlDWxeGn2KdekU,10005
 omdev/amalg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-omdev/amalg/__main__.py,sha256=h94M-VqZ3AFBU2a8zOsjeKK7RF6uINhTHl6OiGbVMgw,163
-omdev/amalg/amalg.py,sha256=qJ-Kllp4mdmx28nt_JTlQnuO-o4XZnTTZP7Uh448b68,19333
+omdev/amalg/__main__.py,sha256=QKvvBlPJalD8Ta6ry7cxChH0lHFj78SKHMMHGUtpFYI,162
+omdev/amalg/gen.py,sha256=wp5CmwMUibcaIJrvwE_gQ1kInhWWgpvRntAI0ONKsQg,5995
+omdev/amalg/imports.py,sha256=KNyuu0zWW63gOSsy4cl1mPs23YVAlICbaF1yEPF147o,2072
+omdev/amalg/main.py,sha256=Ce8apkFfBinNyKVfNutH6ih0tHFc6Ta7RRVyDV5cIMs,4008
+omdev/amalg/manifests.py,sha256=sKnNKsS51zFFD8BIzRUVKtX50dY8pXOtYWi4WHZ_NTU,932
+omdev/amalg/resources.py,sha256=8tQ2yOnGAvPd_YValZZH6_pu9wCY8QeLVBuRUcyScDw,2722
+omdev/amalg/srcfiles.py,sha256=LskAq7kfpH_GuDoNOZF-sxEZiqljDN-VMXxfOfwNfks,3109
+omdev/amalg/strip.py,sha256=lsIkwCle66VjlEFpeQtQ2_sKhpn4HpYN_r4KP6v0DmQ,1439
+omdev/amalg/types.py,sha256=P0VKeWxXKOMQISqHcEFqSBzHQIbIly_nzi1GbMz6EZo,86
+omdev/amalg/typing.py,sha256=q90Mp5JdJaNMLTTwNHSGVx-z6rgfvSZNoV2KRj4dOmg,2049
 omdev/antlr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/antlr/consts.py,sha256=8pR6r0m0P3hAiyiAoJZ-nptd2GYbZ98mxwPL9cpaRuw,279
 omdev/antlr/gen.py,sha256=QnPyVWUrJYqHoOc3XsXA8fJdoiwAj6pyUwdG-DVrIeA,2953
@@ -134,7 +142,7 @@ omdev/toml/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
 omdev/toml/parser.py,sha256=ojhCYIk23ELRx2f9xUCwLTRq13UM6wrYGWoyxZBurlo,29327
 omdev/toml/writer.py,sha256=lk3on3YXVbWuLJa-xsOzOhs1bBAT1vXqw4mBbluZl_w,3040
 omdev/tools/__init__.py,sha256=iVJAOQ0viGTQOm0DLX4uZLro-9jOioYJGLg9s0kDx1A,78
-omdev/tools/cloc.py,sha256=13lUsYLyn1LoelhsCeKrRkIwwPE0x54JmdhXJ_lvWh0,3811
+omdev/tools/cloc.py,sha256=r5HkvaLoGw8djgvGdt_W_CjfSklW585dar9bDhDFeF8,5098
 omdev/tools/doc.py,sha256=iblgUq9_7JZN2i8qmvewrz4OX0paObscBaCj8u77WqI,2555
 omdev/tools/docker.py,sha256=mu0sWnH_L1JjScfWCXXYaux03mcotCS03SD65I93qHI,7384
 omdev/tools/git.py,sha256=zfdPnN-9WSeOQlLoTw5aqAX-UWvz-2p330dx_zaU0WQ,7014
@@ -156,9 +164,9 @@ omdev/tools/json/rendering.py,sha256=jNShMfCpFR9-Kcn6cUFuOChXHjg71diuTC4x7Ofmz-o
 omdev/tools/pawk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/tools/pawk/__main__.py,sha256=VCqeRVnqT1RPEoIrqHFSu4PXVMg4YEgF4qCQm90-eRI,66
 omdev/tools/pawk/pawk.py,sha256=Eckymn22GfychCQcQi96BFqRo_LmiJ-EPhC8TTUJdB4,11446
-omdev-0.0.0.dev163.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev163.dist-info/METADATA,sha256=ZAnRFPdzzNWzA3fjK-vWz97_cfD2z1VaBr1Jp1qNeqc,1760
-omdev-0.0.0.dev163.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-omdev-0.0.0.dev163.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
-omdev-0.0.0.dev163.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev163.dist-info/RECORD,,
+omdev-0.0.0.dev165.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev165.dist-info/METADATA,sha256=djJeiQrUAVjts2OjpttnE69PQiKuV_wm3QIHBEWaS8A,1760
+omdev-0.0.0.dev165.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+omdev-0.0.0.dev165.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
+omdev-0.0.0.dev165.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev165.dist-info/RECORD,,