omdev 0.0.0.dev211__py3-none-any.whl → 0.0.0.dev212__py3-none-any.whl

omdev/.manifests.json

@@ -326,11 +326,25 @@
       }
     }
   },
+  {
+    "module": ".tools.mkenv",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/mkenv.py",
+    "line": 123,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": [
+          "mkenv"
+        ],
+        "mod_name": "omdev.tools.mkenv"
+      }
+    }
+  },
   {
     "module": ".tools.mkrelimp",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/mkrelimp.py",
-    "line": 148,
+    "line": 146,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "py/mkrelimp",

omdev/__about__.py

@@ -45,10 +45,6 @@ class Project(ProjectBase):
             'segno ~= 1.6',
         ],
 
-        'tokens': [
-            'tokenize-rt ~= 6.1',
-        ],
-
         'wheel': [
             'wheel ~= 0.44',
         ],

omdev/amalg/gen.py

@@ -8,11 +8,10 @@ from omlish import collections as col
 from omlish import lang
 from omlish.lite.runtime import LITE_REQUIRED_PYTHON_VERSION
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .srcfiles import SrcFile
 from .srcfiles import make_src_file
 from .strip import strip_main_lines
-from .types import Tokens
 from .typing import Typing
 
 
@@ -71,7 +70,7 @@ class AmalgGenerator:
         return self._src_files()[self._main_path]
 
     @cached.function
-    def _header_lines(self) -> list[Tokens]:
+    def _header_lines(self) -> list[str]:
         header_lines = []
 
         if self._main_file().header_lines:

omdev/amalg/imports.py

@@ -4,8 +4,7 @@ import typing as ta
 
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -22,11 +21,11 @@ class Import:
 
     mod_path: str | None
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def make_import(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
         mounts: ta.Mapping[str, str],
@@ -90,7 +89,7 @@ def make_import(
         as_=as_,
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         mod_path=mod_path,
 

omdev/amalg/manifests.py

@@ -1,17 +1,14 @@
 import ast
 
-import tokenize_rt as trt
-
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
 
 
-def is_manifest_comment(line: Tokens) -> bool:
+def is_manifest_comment(line: tks.Tokens) -> bool:
     if not line:
         return False
 
@@ -22,10 +19,10 @@ def is_manifest_comment(line: Tokens) -> bool:
 
 
 def comment_out_manifest_comment(
-        line: Tokens,
-        cls: list[Tokens],
+        line: tks.Tokens,
+        cls: list[tks.Tokens],
         i: int,
-) -> tuple[list[Tokens], int]:
+) -> tuple[list[tks.Tokens], int]:
     mls = [line]
     while True:
         mls.append(cls[i])
@@ -41,8 +38,8 @@ def comment_out_manifest_comment(
         check.isinstance(check.single(mmod.body), ast.Assign)
         break
 
-    out: list[Tokens] = [
-        [trt.Token('COMMENT', '# ' + tks.join_toks(ml))]
+    out: list[tks.Tokens] = [
+        [tks.Token('COMMENT', '# ' + tks.join_toks(ml))]
         for ml in mls
     ]
 

omdev/amalg/resources.py

@@ -4,10 +4,7 @@ import itertools
 import os.path
 import typing as ta
 
-import tokenize_rt as trt
-
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -19,7 +16,7 @@ class RootLevelResourcesRead(ta.NamedTuple):
     resource: str
 
 
-def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
+def is_root_level_resources_read(lts: tks.Tokens) -> RootLevelResourcesRead | None:
     wts = list(tks.ignore_ws(lts, keep=['INDENT']))
 
     if not tks.match_toks(wts, [
@@ -47,36 +44,36 @@ def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
 def build_resource_lines(
         rsrc: RootLevelResourcesRead,
         path: str,
-) -> list[Tokens]:
+) -> list[tks.Tokens]:
     rf = os.path.join(os.path.dirname(path), rsrc.resource)
 
     if rsrc.kind == 'binary':
         with open(rf, 'rb') as bf:
             rb = bf.read()  # noqa
 
-        out: list[Tokens] = [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='NAME', src='base64'),
-            trt.Token(name='OP', src='.'),
-            trt.Token(name='NAME', src='b64decode'),
-            trt.Token(name='OP', src='('),
-            trt.Token(name='NL', src='\n'),
+        out: list[tks.Tokens] = [[
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='NAME', src='base64'),
+            tks.Token(name='OP', src='.'),
+            tks.Token(name='NAME', src='b64decode'),
+            tks.Token(name='OP', src='('),
+            tks.Token(name='NL', src='\n'),
         ]]
 
         rb64 = base64.b64encode(rb).decode('ascii')
         for chunk in itertools.batched(rb64, 96):
             out.append([
-                trt.Token(name='UNIMPORTANT_WS', src='    '),
-                trt.Token(name='STRING', src=f"'{''.join(chunk)}'"),
-                trt.Token(name='NL', src='\n'),
+                tks.Token(name='UNIMPORTANT_WS', src='    '),
+                tks.Token(name='STRING', src=f"'{''.join(chunk)}'"),
+                tks.Token(name='NL', src='\n'),
             ])
 
         out.append([
-            trt.Token(name='OP', src=')'),
-            trt.Token(name='NEWLINE', src='\n'),
+            tks.Token(name='OP', src=')'),
+            tks.Token(name='NEWLINE', src='\n'),
         ])
 
         return out
@@ -87,12 +84,12 @@ def build_resource_lines(
         rt = rt.replace('\\', '\\\\')  # Escape backslashes
         rt = rt.replace('"""', r'\"\"\"')
         return [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
-            trt.Token(name='NEWLINE', src=''),
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
+            tks.Token(name='NEWLINE', src=''),
         ]]
 
     else:

omdev/amalg/srcfiles.py

@@ -1,12 +1,10 @@
 import dataclasses as dc
 import typing as ta
 
-import tokenize_rt as trt
-
 from omlish import collections as col
 from omlish import lang
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .imports import Import
 from .imports import make_import
 from .manifests import comment_out_manifest_comment
@@ -15,7 +13,6 @@ from .resources import build_resource_lines
 from .resources import is_root_level_resources_read
 from .strip import split_header_lines
 from .strip import strip_header_lines
-from .types import Tokens
 from .typing import Typing
 from .typing import is_root_level_if_type_checking_block
 from .typing import make_typing
@@ -30,13 +27,13 @@ class SrcFile:
     path: str
 
     src: str = dc.field(repr=False)
-    tokens: Tokens = dc.field(repr=False)
-    lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    tokens: tks.Tokens = dc.field(repr=False)
+    lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
-    header_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    header_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
     imports: ta.Sequence[Import] = dc.field(repr=False)
     typings: ta.Sequence[Typing] = dc.field(repr=False)
-    content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    content_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
     ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
 
@@ -51,7 +48,7 @@ def make_src_file(
     with open(path) as f:
         src = f.read().strip()
 
-    tokens = trt.src_to_tokens(src)
+    tokens = tks.src_to_tokens(src)
     lines = tks.split_lines(tokens)
 
     header_lines, cls = split_header_lines(lines)
@@ -61,7 +58,7 @@ def make_src_file(
 
     imps: list[Import] = []
     tys: list[Typing] = []
-    ctls: list[Tokens] = []
+    ctls: list[tks.Tokens] = []
 
     has_binary_resources = False
 

omdev/amalg/strip.py

@@ -2,8 +2,7 @@ import re
 import typing as ta
 
 from .. import magic
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -12,7 +11,7 @@ from .types import Tokens
 HEADER_NAMES = (*tks.WS_NAMES, 'COMMENT', 'STRING')
 
 
-def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[Tokens]]:
+def split_header_lines(lines: ta.Iterable[tks.Tokens]) -> tuple[list[tks.Tokens], list[tks.Tokens]]:
     ws = []
     nws = []
     for line in (it := iter(lines)):
@@ -31,7 +30,7 @@ def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[T
 IF_MAIN_PAT = re.compile(r'if\s+__name__\s+==\s+[\'"]__main__[\'"]\s*:')
 
 
-def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_main_lines(cls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     out = []
 
     for l in (it := iter(cls)):
@@ -59,7 +58,7 @@ STRIPPED_HEADER_PAT = magic.compile_magic_style_pat(
 )
 
 
-def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_header_lines(hls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     if hls and tks.join_toks(hls[0]).startswith('#!'):
         hls = hls[1:]
     out = []

omdev/amalg/types.py

@@ -1,6 +1,6 @@
 import typing as ta
 
-from .. import tokens as tks
+from ..tokens import all as tks
 
 
 Tokens: ta.TypeAlias = tks.Tokens

omdev/amalg/typing.py

@@ -1,7 +1,8 @@
 import dataclasses as dc
 
-from .. import tokens as tks
-from .types import Tokens
+from omlish import check
+
+from ..tokens import all as tks
 
 
 ##
@@ -18,11 +19,11 @@ class Typing:
     src_path: str
     line: int
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def _is_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         exclude_newtypes: bool = False,
 ) -> bool:
@@ -48,7 +49,7 @@ def _is_typing(
 
 
 def make_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
 ) -> Typing | None:
@@ -63,7 +64,7 @@ def make_typing(
         src=tks.join_toks(lts),
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         toks=lts,
     )
@@ -72,7 +73,7 @@ def make_typing(
 ##
 
 
-def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
+def is_root_level_if_type_checking_block(lts: tks.Tokens) -> bool:
     return tks.match_toks(tks.ignore_ws(lts, keep=['INDENT']), [
         ('NAME', 'if'),
         ('NAME', 'ta'),
@@ -83,7 +84,7 @@ def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
 
 
 def skip_root_level_if_type_checking_block(
-        cls: list[Tokens],
+        cls: list[tks.Tokens],
         i: int,
 ) -> int:
     def skip_block():

omdev/ci/cli.py

@@ -112,10 +112,16 @@ class CiCli(ArgparseCli):
         check.state(os.path.isfile(docker_file))
 
         if compose_file is None:
-            compose_file = find_alt_file(
-                'docker/compose.yml',
-                'compose.yml',
-            )
+            compose_file = find_alt_file(*[
+                f'{f}.{x}'
+                for f in [
+                    'docker/docker-compose',
+                    'docker/compose',
+                    'docker-compose',
+                    'compose',
+                ]
+                for x in ['yaml', 'yml']
+            ])
         check.state(os.path.isfile(compose_file))
 
         if not requirements_txts:

omdev/ci/compose.py

@@ -200,7 +200,10 @@ class DockerComposeRun(ExitStacked):
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *itertools.chain.from_iterable(
+                    ['-e', k]
+                    for k in (self._cfg.cmd.env or [])
+                ),
                 *(self._cfg.run_options or []),
                 self._cfg.service,
                 'sh', '-c', shlex.quote(self._cfg.cmd.s),

omdev/ci/requirements.py

@@ -73,8 +73,8 @@ def download_requirements(
             'pip',
             'download',
             '-d', '/requirements',
-            *itertools.chain.from_iterable([
+            *itertools.chain.from_iterable(
                 ['-r', f'/requirements_txt/{os.path.basename(rt)}']
                 for rt in requirements_txts
-            ]),
+            ),
         )

omdev/scripts/ci.py

@@ -2461,7 +2461,10 @@ class DockerComposeRun(ExitStacked):
                 '-f', compose_file,
                 'run',
                 '--rm',
-                *itertools.chain.from_iterable(['-e', k] for k in (self._cfg.cmd.env or [])),
+                *itertools.chain.from_iterable(
+                    ['-e', k]
+                    for k in (self._cfg.cmd.env or [])
+                ),
                 *(self._cfg.run_options or []),
                 self._cfg.service,
                 'sh', '-c', shlex.quote(self._cfg.cmd.s),
@@ -3018,10 +3021,10 @@ def download_requirements(
             'pip',
             'download',
             '-d', '/requirements',
-            *itertools.chain.from_iterable([
+            *itertools.chain.from_iterable(
                 ['-r', f'/requirements_txt/{os.path.basename(rt)}']
                 for rt in requirements_txts
-            ]),
+            ),
         )
 
 
@@ -3358,10 +3361,16 @@ class CiCli(ArgparseCli):
         check.state(os.path.isfile(docker_file))
 
         if compose_file is None:
-            compose_file = find_alt_file(
-                'docker/compose.yml',
-                'compose.yml',
-            )
+            compose_file = find_alt_file(*[
+                f'{f}.{x}'
+                for f in [
+                    'docker/docker-compose',
+                    'docker/compose',
+                    'docker-compose',
+                    'compose',
+                ]
+                for x in ['yaml', 'yml']
+            ])
         check.state(os.path.isfile(compose_file))
 
         if not requirements_txts:

omdev/tokens/all.py

@@ -0,0 +1,35 @@
+from .tokenizert import (  # noqa
+    TokenNames,
+    Token,
+    TokenOffset,
+
+    Tokenization,
+)
+
+from .utils import (  # noqa
+    Tokens,
+
+    WS_NAMES,
+    is_ws,
+    ignore_ws,
+
+    split_lines,
+    join_toks,
+    join_lines,
+
+    match_toks,
+)
+
+
+##
+
+
+ESCAPED_NL = TokenNames.ESCAPED_NL  # noqa
+UNIMPORTANT_WS = TokenNames.UNIMPORTANT_WS  # noqa
+NON_CODING_TOKENS = TokenNames.NON_CODING_TOKENS  # noqa
+
+curly_escape = Tokenization.curly_escape  # noqa
+src_to_tokens = Tokenization.src_to_tokens  # noqa
+parse_string_literal = Tokenization.parse_string_literal  # noqa
+tokens_to_src = Tokenization.tokens_to_src  # noqa
+rfind_string_parts = Tokenization.rfind_string_parts  # noqa

omdev/tokens/tokenizert.py

@@ -0,0 +1,217 @@
+# @omlish-lite
+# ruff: noqa: UP006 UP007
+# Copyright (c) 2017 Anthony Sottile
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# https://github.com/asottile/tokenize-rt/blob/413692b7c1ad8a873caec39dd4f427d55ee538ea/tokenize_rt.py
+import argparse
+import io
+import keyword
+import re
+import tokenize
+import typing as ta
+
+from omlish.lite.check import check
+
+
+##
+
+
+class TokenNames:
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    ESCAPED_NL = 'ESCAPED_NL'
+    UNIMPORTANT_WS = 'UNIMPORTANT_WS'
+    NON_CODING_TOKENS = frozenset(('COMMENT', ESCAPED_NL, 'NL', UNIMPORTANT_WS))
+
+
+class TokenOffset(ta.NamedTuple):
+    line: ta.Optional[int] = None
+    utf8_byte_offset: ta.Optional[int] = None
+
+
+class Token(ta.NamedTuple):
+    name: str
+    src: str
+    line: ta.Optional[int] = None
+    utf8_byte_offset: ta.Optional[int] = None
+
+    @property
+    def offset(self) -> TokenOffset:
+        return TokenOffset(self.line, self.utf8_byte_offset)
+
+    def matches(self, *, name: str, src: str) -> bool:
+        return self.name == name and self.src == src
+
+
+##
+
+
+class Tokenization:
+    _STRING_RE = re.compile('^([^\'"]*)(.*)$', re.DOTALL)
+    _ESCAPED_NL_RE = re.compile(r'\\(\n|\r\n|\r)')
+
+    _NAMED_UNICODE_RE = re.compile(r'(?<!\\)(?:\\\\)*(\\N\{[^}]+\})')
+
+    @classmethod
+    def curly_escape(cls, s: str) -> str:
+        parts = cls._NAMED_UNICODE_RE.split(s)
+        return ''.join(
+            part.replace('{', '{{').replace('}', '}}') if i % 2 == 0 else part
+            for i, part in enumerate(parts)
+        )
+
+    @classmethod
+    def _re_partition(cls, regex: ta.Pattern[str], s: str) -> ta.Tuple[str, str, str]:
+        match = regex.search(s)
+        if match:
+            return s[:match.start()], s[slice(*match.span())], s[match.end():]
+        else:
+            return (s, '', '')
+
+    @classmethod
+    def src_to_tokens(cls, src: str) -> ta.List[Token]:
+        tokenize_target = io.StringIO(src)
+        lines = ('', *tokenize_target)
+
+        tokenize_target.seek(0)
+
+        tokens = []
+        last_line = 1
+        last_col = 0
+        end_offset = 0
+
+        gen = tokenize.generate_tokens(tokenize_target.readline)
+        for tok_type, tok_text, (sline, scol), (eline, ecol), line in gen:
+            if sline > last_line:
+                newtok = lines[last_line][last_col:]
+                for lineno in range(last_line + 1, sline):
+                    newtok += lines[lineno]
+                if scol > 0:
+                    newtok += lines[sline][:scol]
+
+                # a multiline unimportant whitespace may contain escaped newlines
+                while cls._ESCAPED_NL_RE.search(newtok):
+                    ws, nl, newtok = cls._re_partition(cls._ESCAPED_NL_RE, newtok)
+                    if ws:
+                        tokens.append(
+                            Token(TokenNames.UNIMPORTANT_WS, ws, last_line, end_offset),
+                        )
+                        end_offset += len(ws.encode())
+                    tokens.append(Token(TokenNames.ESCAPED_NL, nl, last_line, end_offset))
+                    end_offset = 0
+                    last_line += 1
+                if newtok:
+                    tokens.append(Token(TokenNames.UNIMPORTANT_WS, newtok, sline, 0))
+                    end_offset = len(newtok.encode())
+                else:
+                    end_offset = 0
+
+            elif scol > last_col:
+                newtok = line[last_col:scol]
+                tokens.append(Token(TokenNames.UNIMPORTANT_WS, newtok, sline, end_offset))
+                end_offset += len(newtok.encode())
+
+            tok_name = tokenize.tok_name[tok_type]
+
+            if tok_name == 'FSTRING_MIDDLE':  # pragma: >=3.12 cover
+                if '{' in tok_text or '}' in tok_text:
+                    new_tok_text = cls.curly_escape(tok_text)
+                    ecol += len(new_tok_text) - len(tok_text)
+                    tok_text = new_tok_text
+
+            tokens.append(Token(tok_name, tok_text, sline, end_offset))
+            last_line, last_col = eline, ecol
+            if sline != eline:
+                end_offset = len(lines[last_line][:last_col].encode())
+            else:
+                end_offset += len(tok_text.encode())
+
+        return tokens
+
+    @classmethod
+    def parse_string_literal(cls, src: str) -> ta.Tuple[str, str]:
+        """parse a string literal's source into (prefix, string)"""
+        match = check.not_none(cls._STRING_RE.match(src))
+        return match.group(1), match.group(2)
+
+    @classmethod
+    def tokens_to_src(cls, tokens: ta.Iterable[Token]) -> str:
+        return ''.join(tok.src for tok in tokens)
+
+    @classmethod
+    def rfind_string_parts(cls, tokens: ta.Sequence[Token], start: int) -> ta.Tuple[int, ...]:
+        """
+        Find the indicies of the string parts of a (joined) string literal.
+
+        - `i` should start at the end of the string literal
+        - returns `()` (an empty tuple) for things which are not string literals
+        """
+
+        ret = []
+        depth = 0
+        for i in range(start, -1, -1):
+            token = tokens[i]
+            if token.name == 'STRING':
+                ret.append(i)
+            elif token.name in TokenNames.NON_CODING_TOKENS:
+                pass
+            elif token.src == ')':
+                depth += 1
+            elif depth and token.src == '(':
+                depth -= 1
+                # if we closed the paren(s) make sure it was a parenthesized string
+                # and not actually a call
+                if depth == 0:
+                    for j in range(i - 1, -1, -1):
+                        tok = tokens[j]
+                        if tok.name in TokenNames.NON_CODING_TOKENS:
+                            pass
+                        # this was actually a call and not a parenthesized string
+                        elif (
+                                tok.src in {']', ')'} or (
+                                    tok.name == 'NAME' and
+                                    tok.src not in keyword.kwlist
+                                )
+                        ):
+                            return ()
+                        else:
+                            break
+                    break
+            elif depth:  # it looked like a string but wasn't
+                return ()
+            else:
+                break
+        return tuple(reversed(ret))
+
+
+##
+
+
+if __name__ == '__main__':
+    def main(argv: ta.Optional[ta.Sequence[str]] = None) -> int:
+        parser = argparse.ArgumentParser()
+        parser.add_argument('filename')
+        args = parser.parse_args(argv)
+        with open(args.filename) as f:
+            tokens = Tokenization.src_to_tokens(f.read())
+
+        for token in tokens:
+            line, col = str(token.line), str(token.utf8_byte_offset)
+            print(f'{line}:{col} {token.name} {token.src!r}')
+
+        return 0
+
+    raise SystemExit(main())

omdev/{tokens.py → tokens/utils.py}

@@ -1,16 +1,10 @@
 import itertools
 import typing as ta
 
-from omlish import lang
+from .tokenizert import Token
 
 
-if ta.TYPE_CHECKING:
-    import tokenize_rt as trt
-else:
-    trt = lang.proxy_import('tokenize_rt')
-
-
-Tokens: ta.TypeAlias = ta.Sequence['trt.Token']
+Tokens: ta.TypeAlias = ta.Sequence[Token]
 
 
 ##
@@ -25,15 +19,15 @@ WS_NAMES = (
 )
 
 
-def is_ws(tok: 'trt.Token') -> bool:
+def is_ws(tok: Token) -> bool:
     return tok.name in WS_NAMES
 
 
 def ignore_ws(
-        toks: ta.Iterable['trt.Token'],
+        toks: ta.Iterable[Token],
         *,
         keep: ta.Container[str] = (),
-) -> ta.Iterable['trt.Token']:
+) -> ta.Iterable[Token]:
     return (
         t
         for t in toks
@@ -60,7 +54,7 @@ def join_lines(ls: ta.Iterable[Tokens]) -> str:
 
 
 def match_toks(
-        ts: ta.Iterable['trt.Token'],
+        ts: ta.Iterable[Token],
         pat: ta.Sequence[tuple[str | None, str | tuple[str, ...] | None]],
 ) -> bool:
     it = iter(ts)

omdev/tools/mkenv.py

@@ -0,0 +1,131 @@
+"""
+TODO:
+ - detect file extension
+
+==
+
+export $(./python mkenv.py secrets.yml foo_access_token | xargs)
+eval $(om mkenv -e secrets.yml foo_access_token)
+"""
+import argparse
+import json
+import shlex
+import sys
+import typing as ta
+
+from omlish import check
+from omlish.configs.formats import DEFAULT_CONFIG_FILE_LOADER
+from omlish.specs import jmespath
+
+
+##
+
+
+VALUE_TYPES: tuple[type, ...] = (
+    str,
+    int,
+    float,
+    bool,
+)
+
+
+def extract_item(
+        obj: ta.Any,
+        item: str,
+        *,
+        uppercase_keys: bool = False,
+) -> tuple[str, str]:
+    if '=' not in item:
+        k = item
+        v = obj[k]
+        if uppercase_keys:
+            k = k.upper()
+
+    else:
+        k, p = item.split('=')
+        v = jmespath.search(p, obj)
+
+    #
+
+    if isinstance(v, str):
+        s = v
+
+    elif isinstance(v, bool):
+        s = 'true' if v else 'false'
+
+    else:
+        check.isinstance(v, VALUE_TYPES)
+        s = str(v)
+
+    #
+
+    check.equal(s.strip(), s)
+    for c in '\t\n':
+        check.not_in(c, s)
+
+    #
+
+    return (k, s)
+
+
+def extract_items(
+        obj: ta.Any,
+        items: ta.Iterable[str],
+        **kwargs: ta.Any,
+) -> dict[str, str]:
+    return dict(
+        extract_item(obj, item, **kwargs)
+        for item in items
+    )
+
+
+def _main() -> None:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('file')
+    parser.add_argument('-e', '--for-eval', action='store_true')
+    parser.add_argument('-u', '--uppercase', action='store_true')
+    parser.add_argument('item', nargs='*')
+
+    args = parser.parse_args()
+
+    #
+
+    if args.file == '-':
+        obj = json.loads(sys.stdin.read())
+
+    else:
+        data = DEFAULT_CONFIG_FILE_LOADER.load_file(args.file)
+        obj = data.as_map()
+
+    #
+
+    items = extract_items(
+        obj,
+        args.item,
+        uppercase_keys=args.uppercase,
+    )
+
+    #
+
+    if args.for_eval:
+        cmd = ' '.join([
+            'export',
+            *[f'{k}={qv if (qv := shlex.quote(v)) != v else v}' for k, v in items.items()],
+        ])
+        print(cmd)
+
+    else:
+        for k, v in items.items():
+            print(f'{k}={v}')
+
+
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': ['mkenv'],
+    'mod_name': __name__,
+}}
+
+
+if __name__ == '__main__':
+    _main()

omdev/tools/mkrelimp.py

@@ -4,12 +4,10 @@ import logging
 import os.path
 import typing as ta
 
-import tokenize_rt as trt
-
 from omlish.logs import all as logs
 
-from .. import tokens as tks
 from ..cli import CliModule
+from ..tokens import all as tks
 
 
 T = ta.TypeVar('T')
@@ -91,8 +89,8 @@ class Processor:
         ##
 
         new_tks = list(interleave(
-            trt.Token(name='OP', src='.'),
-            [trt.Token(name='NAME', src=p) for p in rel_imp_name_parts],
+            tks.Token(name='OP', src='.'),
+            [tks.Token(name='NAME', src=p) for p in rel_imp_name_parts],
         ))
         out_tks = [
             *pfx,
@@ -111,7 +109,7 @@ class Processor:
         with open(src_file) as f:
             src = f.read()
 
-        ts = trt.src_to_tokens(src)
+        ts = tks.src_to_tokens(src)
         in_ls = tks.split_lines(ts)
         out_ls = [
             self.process_line_tks(

{omdev-0.0.0.dev211.dist-info → omdev-0.0.0.dev212.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev211
+Version: 0.0.0.dev212
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev211
+Requires-Dist: omlish==0.0.0.dev212
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"
@@ -24,7 +24,6 @@ Requires-Dist: mypy~=1.11; extra == "all"
 Requires-Dist: gprof2dot~=2024.6; extra == "all"
 Requires-Dist: prompt-toolkit~=3.0; extra == "all"
 Requires-Dist: segno~=1.6; extra == "all"
-Requires-Dist: tokenize-rt~=6.1; extra == "all"
 Requires-Dist: wheel~=0.44; extra == "all"
 Provides-Extra: black
 Requires-Dist: black~=24.10; extra == "black"
@@ -43,7 +42,5 @@ Provides-Extra: ptk
 Requires-Dist: prompt-toolkit~=3.0; extra == "ptk"
 Provides-Extra: qr
 Requires-Dist: segno~=1.6; extra == "qr"
-Provides-Extra: tokens
-Requires-Dist: tokenize-rt~=6.1; extra == "tokens"
 Provides-Extra: wheel
 Requires-Dist: wheel~=0.44; extra == "wheel"

{omdev-0.0.0.dev211.dist-info → omdev-0.0.0.dev212.dist-info}/RECORD

@@ -1,5 +1,5 @@
-omdev/.manifests.json,sha256=bU6NL_9c9hikH-GGCq_JL5C_NWilB3fvv-UQgvJoeZw,8820
-omdev/__about__.py,sha256=n5x-SO70OgbDQFzQ1d7sZDVMsnkQc4PxQZPFaIQFa0E,1281
+omdev/.manifests.json,sha256=kIHxCLwrqvBumPKT0GKsgRPhrJSovHf9jDnWzUDZMHs,9092
+omdev/__about__.py,sha256=j3vFclhFvyPICV6FK4aDApFzMCqJWxv9FaWwdwXrSgw,1215
 omdev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/bracepy.py,sha256=I8EdqtDvxzAi3I8TuMEW-RBfwXfqKbwp06CfOdj3L1o,2743
 omdev/classdot.py,sha256=YOvgy6x295I_8NKBbBlRVd3AN7Osirm_Lqt4Wj0j9rY,1631
@@ -10,19 +10,18 @@ omdev/pip.py,sha256=7cZ_IOpekQvgPm_gKnX3Pr8xjqUid50PPScTlZCYVlM,2118
 omdev/revisions.py,sha256=0feRWC0Uttd6K9cCImAHEXoo6-Nuso3tpaHUuhzBlRQ,4985
 omdev/secrets.py,sha256=bcquaBIDKqX4UIKOzUuKrX7nxVCenj67rRHIMIrd9bk,540
 omdev/tagstrings.py,sha256=hrinoRmYCFMt4WYCZAYrocVYKQvIApNGKbJaGz8whqs,5334
-omdev/tokens.py,sha256=zh2TCAfCbcq8ZnoVdQ824jrTiwNy3XJ_oCqlZpLpcCY,1574
 omdev/wheelfile.py,sha256=yfupGcGkbFlmzGzKU64k_vmOKpaKnUlDWxeGn2KdekU,10005
 omdev/amalg/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/amalg/__main__.py,sha256=1sZH8SLAueWxMxK9ngvndUW3L_rw7f-s_jK3ZP1yAH8,170
-omdev/amalg/gen.py,sha256=wp5CmwMUibcaIJrvwE_gQ1kInhWWgpvRntAI0ONKsQg,5995
-omdev/amalg/imports.py,sha256=KNyuu0zWW63gOSsy4cl1mPs23YVAlICbaF1yEPF147o,2072
+omdev/amalg/gen.py,sha256=auSK_NakPiEW-8pLr0JnE6hg995zOmQZaAJh38v1o5Q,5969
+omdev/amalg/imports.py,sha256=me-I_FOD5874NVebZGt_wlc9kxEpuZV-XGuifbNkgNk,2073
 omdev/amalg/main.py,sha256=CNkpAMVXlBoaCmgs68rKSUlL8Xh4AuNdZjSFZPeao_4,4313
-omdev/amalg/manifests.py,sha256=sKnNKsS51zFFD8BIzRUVKtX50dY8pXOtYWi4WHZ_NTU,932
-omdev/amalg/resources.py,sha256=wldsAHeMaoUbL7bicE-Ty9QIm1Q-1VllA1Qd3juRFXo,2730
-omdev/amalg/srcfiles.py,sha256=LskAq7kfpH_GuDoNOZF-sxEZiqljDN-VMXxfOfwNfks,3109
-omdev/amalg/strip.py,sha256=lsIkwCle66VjlEFpeQtQ2_sKhpn4HpYN_r4KP6v0DmQ,1439
-omdev/amalg/types.py,sha256=P0VKeWxXKOMQISqHcEFqSBzHQIbIly_nzi1GbMz6EZo,86
-omdev/amalg/typing.py,sha256=q90Mp5JdJaNMLTTwNHSGVx-z6rgfvSZNoV2KRj4dOmg,2049
+omdev/amalg/manifests.py,sha256=CGwh-DiOU4ZKv0p_uUvdpBl2H72g468kvBHtguOHGMA,902
+omdev/amalg/resources.py,sha256=aIqVd1tbGCv4vIt6gbzqnB0L3PsQsAeqVaZJ_J7QSz8,2692
+omdev/amalg/srcfiles.py,sha256=TKSYSfv78Wb8bcsQS3V76dV_H_MICPwzv06Ar09ew9U,3079
+omdev/amalg/strip.py,sha256=dWQQ5WbtcebLi_PGjPzVYey2mJOnrRES8rkoUS8L52w,1444
+omdev/amalg/types.py,sha256=BXXJI0VctKTsZv_wXiyMMq3-xShxZ1ak0wxXUK8n9_g,89
+omdev/amalg/typing.py,sha256=viaoxX4ijxf1dfrgL5W0jwYxSzFouNhmdUKi-9uL3KU,2088
 omdev/antlr/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/antlr/consts.py,sha256=6iKuncbRXnGAqemNAceaR2mdFSs-8VbBV-pluM-8Bu4,284
 omdev/antlr/gen.py,sha256=ZuDoh8ksIGlTSufEnJXogpk7gRVfmPirERdLx2nMWDo,3020
@@ -74,10 +73,10 @@ omdev/ci/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/ci/__main__.py,sha256=Jsrv3P7LX2Cg08W7ByZfZ1JQT4lgLDPW1qNAmShFuMk,75
 omdev/ci/cache.py,sha256=jGrsnYNHnnOLJGIEoyHJUdpVzCBdh4tmy2d22Xgarnk,4132
 omdev/ci/ci.py,sha256=4ZbFrXL22NQLvOZydrUo-MgaW8I6u-hT-N57wDHakyI,7396
-omdev/ci/cli.py,sha256=8y7Da3cPtlBrezXQ8393tVGFxoelngSxvCkDEKeK50w,5149
-omdev/ci/compose.py,sha256=mUViov0mzj5ffAupl72TNkUdpBC2gB_hwG9aQmP6rAY,5082
+omdev/ci/cli.py,sha256=-hxHvbHqPRtm4rhy1ZOTjP_eo5rEYHEDBcWdJqxhVUw,5348
+omdev/ci/compose.py,sha256=IFZSgBdqdrdDf2z3MMDYJU1q-z5-TEvZW6K0CtwNi5s,5140
 omdev/ci/docker.py,sha256=BsKuqRHYSstqC1sQXnbHevNGUBWf5M9PYLobjZO7UMk,3218
-omdev/ci/requirements.py,sha256=vZR1rjBUkhGNjXSfFZgMYhSHKAgcsbONqbCuUB5dqVo,2117
+omdev/ci/requirements.py,sha256=9K-f3DXZANoOonLjCCtpE6wR8JPJxj33ouo0VKNjZUE,2115
 omdev/ci/shell.py,sha256=KTkVZb_piQeB5Z12M2jUS_zGJfLR_Y2W2uYkrQpJpyw,850
 omdev/ci/utils.py,sha256=yiLSFOy4nNN9EY1C8rBe8fXdTmjpfBb6n1IgpzVJ00g,1605
 omdev/ci/github/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -172,7 +171,7 @@ omdev/pyproject/resources/docker-dev.sh,sha256=DHkz5D18jok_oDolfg2mqrvGRWFoCe9GQ
 omdev/pyproject/resources/python.sh,sha256=jvrwddYw2KNttpuImLbdCdJK0HsUNMrHtTnmLvhxQxg,757
 omdev/scripts/__init__.py,sha256=MKCvUAEQwsIvwLixwtPlpBqmkMXLCnjjXyAXvVpDwVk,91
 omdev/scripts/bumpversion.py,sha256=Kn7fo73Hs8uJh3Hi3EIyLOlzLPWAC6dwuD_lZ3cIzuY,1064
-omdev/scripts/ci.py,sha256=52RkuVmRKgqcfAXaBA4sb1zgD8v8tAsPRaZ6LVxF7VM,87852
+omdev/scripts/ci.py,sha256=BIWoEqc-AWnjItuADz4b_ufhk7aySwKptCe8nitBzhU,88107
 omdev/scripts/execrss.py,sha256=mR0G0wERBYtQmVIn63lCIIFb5zkCM6X_XOENDFYDBKc,651
 omdev/scripts/exectime.py,sha256=sFb376GflU6s9gNX-2-we8hgH6w5MuQNS9g6i4SqJIo,610
 omdev/scripts/importtrace.py,sha256=oa7CtcWJVMNDbyIEiRHej6ICfABfErMeo4_haIqe18Q,14041
@@ -180,6 +179,10 @@ omdev/scripts/interp.py,sha256=EB8hRemH41IMY_Iho_tZdIYdP0i5Q5z4a-oWr7K4bLk,14126
 omdev/scripts/pyproject.py,sha256=4KmTDZFqeTpNAdhW0DXTzmrH4LjHlp4rE6mRzhnBV1s,243089
 omdev/scripts/slowcat.py,sha256=lssv4yrgJHiWfOiHkUut2p8E8Tq32zB-ujXESQxFFHY,2728
 omdev/scripts/tmpexec.py,sha256=WTYcf56Tj2qjYV14AWmV8SfT0u6Y8eIU6cKgQRvEK3c,1442
+omdev/tokens/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+omdev/tokens/all.py,sha256=4t_3NnU-ph_ehyC3XLB_J_lIda7KD1Xvc7rKexOVBAo,686
+omdev/tokens/tokenizert.py,sha256=y2sFFLJzcYXqP2xiO-6zmJuRYfA02-CGU3CPeD1NXjg,7895
+omdev/tokens/utils.py,sha256=DJA2jsDzrtQxnO0bdkUk1-GmenX4ECugdhl1xOBe7QI,1448
 omdev/tools/__init__.py,sha256=iVJAOQ0viGTQOm0DLX4uZLro-9jOioYJGLg9s0kDx1A,78
 omdev/tools/cloc.py,sha256=jYlMHBae9oGKN4VKeBGuqjiQNcM2be7KIoTF0oNwx_I,5205
 omdev/tools/doc.py,sha256=wvgGhv6aFaV-Zl-Qivejx37i-lKQ207rZ-4K2fPf-Ss,2547
@@ -187,7 +190,8 @@ omdev/tools/docker.py,sha256=KVFckA8eAdiapFUr8xkfMw9Uv3Qy4oNq0e70Lqt1F7I,7352
 omdev/tools/git.py,sha256=fiQc4w2w63PELLLyMXdwpmgpzFzs7UNn35vewWckScM,7514
 omdev/tools/importscan.py,sha256=nhJIhtjDY6eFVlReP7fegvv6L5ZjN-Z2VeyhsBonev4,4639
 omdev/tools/linehisto.py,sha256=0ZNm34EuiZBE9Q2YC6KNLNNydNT8QPSOwvYzXiU9S2Q,8881
-omdev/tools/mkrelimp.py,sha256=kyu_BbUakKHEEOxNEvYWk7tH1ixCfVb3NqqT8U-BozE,4066
+omdev/tools/mkenv.py,sha256=G2tu5bmiyKFyZuqtUoM7Z-6AI6CI86F2LwoIozoWOvo,2300
+omdev/tools/mkrelimp.py,sha256=p_35JBGDec0gvD6AHQ279iHcjU39E7Si6LcEDCMrJpo,4042
 omdev/tools/notebook.py,sha256=q1YMGwM1skHv-dPbtT_cM7UOGFNiMEAxjr6rr6rbobk,3494
 omdev/tools/pip.py,sha256=eBD41hp-V3thGfhUBM3Erxl4CSG-5LG6Szo1sA76P2k,3459
 omdev/tools/prof.py,sha256=hQakAsViJD4gLJpLLZnTkOqmTDAwM48Nx5q-O_aFlYM,1467
@@ -204,9 +208,9 @@ omdev/tools/json/rendering.py,sha256=tMcjOW5edfozcMSTxxvF7WVTsbYLoe9bCKFh50qyaGw
 omdev/tools/pawk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 omdev/tools/pawk/__main__.py,sha256=VCqeRVnqT1RPEoIrqHFSu4PXVMg4YEgF4qCQm90-eRI,66
 omdev/tools/pawk/pawk.py,sha256=zsEkfQX0jF5bn712uqPAyBSdJt2dno1LH2oeSMNfXQI,11424
-omdev-0.0.0.dev211.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
-omdev-0.0.0.dev211.dist-info/METADATA,sha256=DEQf4B4GmTSCD3ojKQ0FEypSgogbvXVC9taoW9pCnpM,1760
-omdev-0.0.0.dev211.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-omdev-0.0.0.dev211.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
-omdev-0.0.0.dev211.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
-omdev-0.0.0.dev211.dist-info/RECORD,,
+omdev-0.0.0.dev212.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
+omdev-0.0.0.dev212.dist-info/METADATA,sha256=3Jk_ZtfrtW8Z3LA2VBaaQly2E6XVpPso8BKuL1m0bfI,1638
+omdev-0.0.0.dev212.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+omdev-0.0.0.dev212.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
+omdev-0.0.0.dev212.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
+omdev-0.0.0.dev212.dist-info/RECORD,,