omdev 0.0.0.dev211__tar.gz → 0.0.0.dev212__tar.gz

{omdev-0.0.0.dev211/omdev.egg-info → omdev-0.0.0.dev212}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev211
+Version: 0.0.0.dev212
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev211
+Requires-Dist: omlish==0.0.0.dev212
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"
@@ -24,7 +24,6 @@ Requires-Dist: mypy~=1.11; extra == "all"
 Requires-Dist: gprof2dot~=2024.6; extra == "all"
 Requires-Dist: prompt-toolkit~=3.0; extra == "all"
 Requires-Dist: segno~=1.6; extra == "all"
-Requires-Dist: tokenize-rt~=6.1; extra == "all"
 Requires-Dist: wheel~=0.44; extra == "all"
 Provides-Extra: black
 Requires-Dist: black~=24.10; extra == "black"
@@ -43,7 +42,5 @@ Provides-Extra: ptk
 Requires-Dist: prompt-toolkit~=3.0; extra == "ptk"
 Provides-Extra: qr
 Requires-Dist: segno~=1.6; extra == "qr"
-Provides-Extra: tokens
-Requires-Dist: tokenize-rt~=6.1; extra == "tokens"
 Provides-Extra: wheel
 Requires-Dist: wheel~=0.44; extra == "wheel"

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/.manifests.json

@@ -326,11 +326,25 @@
       }
     }
   },
+  {
+    "module": ".tools.mkenv",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/mkenv.py",
+    "line": 123,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": [
+          "mkenv"
+        ],
+        "mod_name": "omdev.tools.mkenv"
+      }
+    }
+  },
   {
     "module": ".tools.mkrelimp",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/mkrelimp.py",
-    "line": 148,
+    "line": 146,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "py/mkrelimp",

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/__about__.py

@@ -45,10 +45,6 @@ class Project(ProjectBase):
             'segno ~= 1.6',
         ],
 
-        'tokens': [
-            'tokenize-rt ~= 6.1',
-        ],
-
         'wheel': [
             'wheel ~= 0.44',
         ],

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/gen.py

@@ -8,11 +8,10 @@ from omlish import collections as col
 from omlish import lang
 from omlish.lite.runtime import LITE_REQUIRED_PYTHON_VERSION
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .srcfiles import SrcFile
 from .srcfiles import make_src_file
 from .strip import strip_main_lines
-from .types import Tokens
 from .typing import Typing
 
 
@@ -71,7 +70,7 @@ class AmalgGenerator:
         return self._src_files()[self._main_path]
 
     @cached.function
-    def _header_lines(self) -> list[Tokens]:
+    def _header_lines(self) -> list[str]:
         header_lines = []
 
         if self._main_file().header_lines:

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/imports.py

@@ -4,8 +4,7 @@ import typing as ta
 
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -22,11 +21,11 @@ class Import:
 
     mod_path: str | None
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def make_import(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
         mounts: ta.Mapping[str, str],
@@ -90,7 +89,7 @@ def make_import(
         as_=as_,
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         mod_path=mod_path,
 

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/manifests.py

@@ -1,17 +1,14 @@
 import ast
 
-import tokenize_rt as trt
-
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
 
 
-def is_manifest_comment(line: Tokens) -> bool:
+def is_manifest_comment(line: tks.Tokens) -> bool:
     if not line:
         return False
 
@@ -22,10 +19,10 @@ def is_manifest_comment(line: Tokens) -> bool:
 
 
 def comment_out_manifest_comment(
-        line: Tokens,
-        cls: list[Tokens],
+        line: tks.Tokens,
+        cls: list[tks.Tokens],
         i: int,
-) -> tuple[list[Tokens], int]:
+) -> tuple[list[tks.Tokens], int]:
     mls = [line]
     while True:
         mls.append(cls[i])
@@ -41,8 +38,8 @@ def comment_out_manifest_comment(
         check.isinstance(check.single(mmod.body), ast.Assign)
         break
 
-    out: list[Tokens] = [
-        [trt.Token('COMMENT', '# ' + tks.join_toks(ml))]
+    out: list[tks.Tokens] = [
+        [tks.Token('COMMENT', '# ' + tks.join_toks(ml))]
         for ml in mls
     ]
 

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/resources.py

@@ -4,10 +4,7 @@ import itertools
 import os.path
 import typing as ta
 
-import tokenize_rt as trt
-
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -19,7 +16,7 @@ class RootLevelResourcesRead(ta.NamedTuple):
     resource: str
 
 
-def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
+def is_root_level_resources_read(lts: tks.Tokens) -> RootLevelResourcesRead | None:
     wts = list(tks.ignore_ws(lts, keep=['INDENT']))
 
     if not tks.match_toks(wts, [
@@ -47,36 +44,36 @@ def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
 def build_resource_lines(
         rsrc: RootLevelResourcesRead,
         path: str,
-) -> list[Tokens]:
+) -> list[tks.Tokens]:
     rf = os.path.join(os.path.dirname(path), rsrc.resource)
 
     if rsrc.kind == 'binary':
         with open(rf, 'rb') as bf:
             rb = bf.read()  # noqa
 
-        out: list[Tokens] = [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='NAME', src='base64'),
-            trt.Token(name='OP', src='.'),
-            trt.Token(name='NAME', src='b64decode'),
-            trt.Token(name='OP', src='('),
-            trt.Token(name='NL', src='\n'),
+        out: list[tks.Tokens] = [[
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='NAME', src='base64'),
+            tks.Token(name='OP', src='.'),
+            tks.Token(name='NAME', src='b64decode'),
+            tks.Token(name='OP', src='('),
+            tks.Token(name='NL', src='\n'),
         ]]
 
         rb64 = base64.b64encode(rb).decode('ascii')
         for chunk in itertools.batched(rb64, 96):
             out.append([
-                trt.Token(name='UNIMPORTANT_WS', src='    '),
-                trt.Token(name='STRING', src=f"'{''.join(chunk)}'"),
-                trt.Token(name='NL', src='\n'),
+                tks.Token(name='UNIMPORTANT_WS', src='    '),
+                tks.Token(name='STRING', src=f"'{''.join(chunk)}'"),
+                tks.Token(name='NL', src='\n'),
             ])
 
         out.append([
-            trt.Token(name='OP', src=')'),
-            trt.Token(name='NEWLINE', src='\n'),
+            tks.Token(name='OP', src=')'),
+            tks.Token(name='NEWLINE', src='\n'),
         ])
 
         return out
@@ -87,12 +84,12 @@ def build_resource_lines(
         rt = rt.replace('\\', '\\\\')  # Escape backslashes
         rt = rt.replace('"""', r'\"\"\"')
         return [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
-            trt.Token(name='NEWLINE', src=''),
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
+            tks.Token(name='NEWLINE', src=''),
         ]]
 
     else:

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/srcfiles.py

@@ -1,12 +1,10 @@
 import dataclasses as dc
 import typing as ta
 
-import tokenize_rt as trt
-
 from omlish import collections as col
 from omlish import lang
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .imports import Import
 from .imports import make_import
 from .manifests import comment_out_manifest_comment
@@ -15,7 +13,6 @@ from .resources import build_resource_lines
 from .resources import is_root_level_resources_read
 from .strip import split_header_lines
 from .strip import strip_header_lines
-from .types import Tokens
 from .typing import Typing
 from .typing import is_root_level_if_type_checking_block
 from .typing import make_typing
@@ -30,13 +27,13 @@ class SrcFile:
     path: str
 
     src: str = dc.field(repr=False)
-    tokens: Tokens = dc.field(repr=False)
-    lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    tokens: tks.Tokens = dc.field(repr=False)
+    lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
-    header_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    header_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
     imports: ta.Sequence[Import] = dc.field(repr=False)
     typings: ta.Sequence[Typing] = dc.field(repr=False)
-    content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    content_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
     ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
 
@@ -51,7 +48,7 @@ def make_src_file(
     with open(path) as f:
         src = f.read().strip()
 
-    tokens = trt.src_to_tokens(src)
+    tokens = tks.src_to_tokens(src)
     lines = tks.split_lines(tokens)
 
     header_lines, cls = split_header_lines(lines)
@@ -61,7 +58,7 @@ def make_src_file(
 
     imps: list[Import] = []
     tys: list[Typing] = []
-    ctls: list[Tokens] = []
+    ctls: list[tks.Tokens] = []
 
     has_binary_resources = False
 

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/strip.py

@@ -2,8 +2,7 @@ import re
 import typing as ta
 
 from .. import magic
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -12,7 +11,7 @@ from .types import Tokens
 HEADER_NAMES = (*tks.WS_NAMES, 'COMMENT', 'STRING')
 
 
-def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[Tokens]]:
+def split_header_lines(lines: ta.Iterable[tks.Tokens]) -> tuple[list[tks.Tokens], list[tks.Tokens]]:
     ws = []
     nws = []
     for line in (it := iter(lines)):
@@ -31,7 +30,7 @@ def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[T
 IF_MAIN_PAT = re.compile(r'if\s+__name__\s+==\s+[\'"]__main__[\'"]\s*:')
 
 
-def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_main_lines(cls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     out = []
 
     for l in (it := iter(cls)):
@@ -59,7 +58,7 @@ STRIPPED_HEADER_PAT = magic.compile_magic_style_pat(
 )
 
 
-def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_header_lines(hls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     if hls and tks.join_toks(hls[0]).startswith('#!'):
         hls = hls[1:]
     out = []

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/types.py

@@ -1,6 +1,6 @@
 import typing as ta
 
-from .. import tokens as tks
+from ..tokens import all as tks
 
 
 Tokens: ta.TypeAlias = tks.Tokens

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/amalg/typing.py

@@ -1,7 +1,8 @@
 import dataclasses as dc
 
-from .. import tokens as tks
-from .types import Tokens
+from omlish import check
+
+from ..tokens import all as tks
 
 
 ##
@@ -18,11 +19,11 @@ class Typing:
     src_path: str
     line: int
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def _is_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         exclude_newtypes: bool = False,
 ) -> bool:
@@ -48,7 +49,7 @@ def _is_typing(
 
 
 def make_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
 ) -> Typing | None:
@@ -63,7 +64,7 @@ def make_typing(
         src=tks.join_toks(lts),
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         toks=lts,
     )
@@ -72,7 +73,7 @@ def make_typing(
 ##
 
 
-def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
+def is_root_level_if_type_checking_block(lts: tks.Tokens) -> bool:
     return tks.match_toks(tks.ignore_ws(lts, keep=['INDENT']), [
         ('NAME', 'if'),
         ('NAME', 'ta'),
@@ -83,7 +84,7 @@ def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
 
 
 def skip_root_level_if_type_checking_block(
-        cls: list[Tokens],
+        cls: list[tks.Tokens],
         i: int,
 ) -> int:
     def skip_block():

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/ci/cli.py

@@ -112,10 +112,16 @@ class CiCli(ArgparseCli):
         check.state(os.path.isfile(docker_file))
 
         if compose_file is None:
-            compose_file = find_alt_file(
-                'docker/compose.yml',
-                'compose.yml',
-            )
+            compose_file = find_alt_file(*[
+                f'{f}.{x}'
+                for f in [
+                    'docker/docker-compose',
+                    'docker/compose',
+                    'docker-compose',
+                    'compose',
+                ]
+                for x in ['yaml', 'yml']
+            ])
         check.state(os.path.isfile(compose_file))
 
         if not requirements_txts:

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/ci/compose.py

@@ -200,7 +200,10 @@ class DockerComposeRun(ExitStacked):
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *itertools.chain.from_iterable(
+                    ['-e', k]
+                    for k in (self._cfg.cmd.env or [])
+                ),
                 *(self._cfg.run_options or []),
                 self._cfg.service,
                 'sh', '-c', shlex.quote(self._cfg.cmd.s),

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/ci/requirements.py

@@ -73,8 +73,8 @@ def download_requirements(
             'pip',
             'download',
             '-d', '/requirements',
-            *itertools.chain.from_iterable([
+            *itertools.chain.from_iterable(
                 ['-r', f'/requirements_txt/{os.path.basename(rt)}']
                 for rt in requirements_txts
-            ]),
+            ),
         )

{omdev-0.0.0.dev211 → omdev-0.0.0.dev212}/omdev/scripts/ci.py

@@ -2461,7 +2461,10 @@ class DockerComposeRun(ExitStacked):
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *itertools.chain.from_iterable(
+                    ['-e', k]
+                    for k in (self._cfg.cmd.env or [])
+                ),
                 *(self._cfg.run_options or []),
                 self._cfg.service,
                 'sh', '-c', shlex.quote(self._cfg.cmd.s),
@@ -3018,10 +3021,10 @@ def download_requirements(
             'pip',
             'download',
             '-d', '/requirements',
-            *itertools.chain.from_iterable([
+            *itertools.chain.from_iterable(
                 ['-r', f'/requirements_txt/{os.path.basename(rt)}']
                 for rt in requirements_txts
-            ]),
+            ),
         )
 
 
@@ -3358,10 +3361,16 @@ class CiCli(ArgparseCli):
         check.state(os.path.isfile(docker_file))
 
         if compose_file is None:
-            compose_file = find_alt_file(
-                'docker/compose.yml',
-                'compose.yml',
-            )
+            compose_file = find_alt_file(*[
+                f'{f}.{x}'
+                for f in [
+                    'docker/docker-compose',
+                    'docker/compose',
+                    'docker-compose',
+                    'compose',
+                ]
+                for x in ['yaml', 'yml']
+            ])
         check.state(os.path.isfile(compose_file))
 
         if not requirements_txts:

omdev-0.0.0.dev212/omdev/tokens/all.py

@@ -0,0 +1,35 @@
+from .tokenizert import (  # noqa
+    TokenNames,
+    Token,
+    TokenOffset,
+
+    Tokenization,
+)
+
+from .utils import (  # noqa
+    Tokens,
+
+    WS_NAMES,
+    is_ws,
+    ignore_ws,
+
+    split_lines,
+    join_toks,
+    join_lines,
+
+    match_toks,
+)
+
+
+##
+
+
+ESCAPED_NL = TokenNames.ESCAPED_NL  # noqa
+UNIMPORTANT_WS = TokenNames.UNIMPORTANT_WS  # noqa
+NON_CODING_TOKENS = TokenNames.NON_CODING_TOKENS  # noqa
+
+curly_escape = Tokenization.curly_escape  # noqa
+src_to_tokens = Tokenization.src_to_tokens  # noqa
+parse_string_literal = Tokenization.parse_string_literal  # noqa
+tokens_to_src = Tokenization.tokens_to_src  # noqa
+rfind_string_parts = Tokenization.rfind_string_parts  # noqa