omdev 0.0.0.dev210__py3-none-any.whl → 0.0.0.dev212__py3-none-any.whl

omdev/scripts/interp.py

@@ -1189,7 +1189,7 @@ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
 ##
 
 
-def camel_case(name: str, lower: bool = False) -> str:
+def camel_case(name: str, *, lower: bool = False) -> str:
     if not name:
         return ''
     s = ''.join(map(str.capitalize, name.split('_')))  # noqa
@@ -1206,6 +1206,27 @@ def snake_case(name: str) -> str:
 ##
 
 
+def is_dunder(name: str) -> bool:
+    return (
+        name[:2] == name[-2:] == '__' and
+        name[2:3] != '_' and
+        name[-3:-2] != '_' and
+        len(name) > 4
+    )
+
+
+def is_sunder(name: str) -> bool:
+    return (
+        name[0] == name[-1] == '_' and
+        name[1:2] != '_' and
+        name[-2:-1] != '_' and
+        len(name) > 2
+    )
+
+
+##
+
+
 def strip_with_newline(s: str) -> str:
     if not s:
         return ''
@@ -1237,27 +1258,6 @@ def split_keep_delimiter(s, d):
 ##
 
 
-def is_dunder(name: str) -> bool:
-    return (
-        name[:2] == name[-2:] == '__' and
-        name[2:3] != '_' and
-        name[-3:-2] != '_' and
-        len(name) > 4
-    )
-
-
-def is_sunder(name: str) -> bool:
-    return (
-        name[0] == name[-1] == '_' and
-        name[1:2] != '_' and
-        name[-2:-1] != '_' and
-        len(name) > 2
-    )
-
-
-##
-
-
 def attr_repr(obj: ta.Any, *attrs: str) -> str:
     return f'{type(obj).__name__}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
 

omdev/scripts/pyproject.py

@@ -2497,7 +2497,7 @@ def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
 ##
 
 
-def camel_case(name: str, lower: bool = False) -> str:
+def camel_case(name: str, *, lower: bool = False) -> str:
     if not name:
         return ''
     s = ''.join(map(str.capitalize, name.split('_')))  # noqa
@@ -2514,6 +2514,27 @@ def snake_case(name: str) -> str:
 ##
 
 
+def is_dunder(name: str) -> bool:
+    return (
+        name[:2] == name[-2:] == '__' and
+        name[2:3] != '_' and
+        name[-3:-2] != '_' and
+        len(name) > 4
+    )
+
+
+def is_sunder(name: str) -> bool:
+    return (
+        name[0] == name[-1] == '_' and
+        name[1:2] != '_' and
+        name[-2:-1] != '_' and
+        len(name) > 2
+    )
+
+
+##
+
+
 def strip_with_newline(s: str) -> str:
     if not s:
         return ''
@@ -2545,27 +2566,6 @@ def split_keep_delimiter(s, d):
 ##
 
 
-def is_dunder(name: str) -> bool:
-    return (
-        name[:2] == name[-2:] == '__' and
-        name[2:3] != '_' and
-        name[-3:-2] != '_' and
-        len(name) > 4
-    )
-
-
-def is_sunder(name: str) -> bool:
-    return (
-        name[0] == name[-1] == '_' and
-        name[1:2] != '_' and
-        name[-2:-1] != '_' and
-        len(name) > 2
-    )
-
-
-##
-
-
 def attr_repr(obj: ta.Any, *attrs: str) -> str:
     return f'{type(obj).__name__}({", ".join(f"{attr}={getattr(obj, attr)!r}" for attr in attrs)})'
 

omdev/tokens/all.py

@@ -0,0 +1,35 @@
+from .tokenizert import (  # noqa
+    TokenNames,
+    Token,
+    TokenOffset,
+
+    Tokenization,
+)
+
+from .utils import (  # noqa
+    Tokens,
+
+    WS_NAMES,
+    is_ws,
+    ignore_ws,
+
+    split_lines,
+    join_toks,
+    join_lines,
+
+    match_toks,
+)
+
+
+##
+
+
+ESCAPED_NL = TokenNames.ESCAPED_NL  # noqa
+UNIMPORTANT_WS = TokenNames.UNIMPORTANT_WS  # noqa
+NON_CODING_TOKENS = TokenNames.NON_CODING_TOKENS  # noqa
+
+curly_escape = Tokenization.curly_escape  # noqa
+src_to_tokens = Tokenization.src_to_tokens  # noqa
+parse_string_literal = Tokenization.parse_string_literal  # noqa
+tokens_to_src = Tokenization.tokens_to_src  # noqa
+rfind_string_parts = Tokenization.rfind_string_parts  # noqa

omdev/tokens/tokenizert.py

@@ -0,0 +1,217 @@
+# @omlish-lite
+# ruff: noqa: UP006 UP007
+# Copyright (c) 2017 Anthony Sottile
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# https://github.com/asottile/tokenize-rt/blob/413692b7c1ad8a873caec39dd4f427d55ee538ea/tokenize_rt.py
+import argparse
+import io
+import keyword
+import re
+import tokenize
+import typing as ta
+
+from omlish.lite.check import check
+
+
+##
+
+
+class TokenNames:
+    def __new__(cls, *args, **kwargs):  # noqa
+        raise TypeError
+
+    ESCAPED_NL = 'ESCAPED_NL'
+    UNIMPORTANT_WS = 'UNIMPORTANT_WS'
+    NON_CODING_TOKENS = frozenset(('COMMENT', ESCAPED_NL, 'NL', UNIMPORTANT_WS))
+
+
+class TokenOffset(ta.NamedTuple):
+    line: ta.Optional[int] = None
+    utf8_byte_offset: ta.Optional[int] = None
+
+
+class Token(ta.NamedTuple):
+    name: str
+    src: str
+    line: ta.Optional[int] = None
+    utf8_byte_offset: ta.Optional[int] = None
+
+    @property
+    def offset(self) -> TokenOffset:
+        return TokenOffset(self.line, self.utf8_byte_offset)
+
+    def matches(self, *, name: str, src: str) -> bool:
+        return self.name == name and self.src == src
+
+
+##
+
+
+class Tokenization:
+    _STRING_RE = re.compile('^([^\'"]*)(.*)$', re.DOTALL)
+    _ESCAPED_NL_RE = re.compile(r'\\(\n|\r\n|\r)')
+
+    _NAMED_UNICODE_RE = re.compile(r'(?<!\\)(?:\\\\)*(\\N\{[^}]+\})')
+
+    @classmethod
+    def curly_escape(cls, s: str) -> str:
+        parts = cls._NAMED_UNICODE_RE.split(s)
+        return ''.join(
+            part.replace('{', '{{').replace('}', '}}') if i % 2 == 0 else part
+            for i, part in enumerate(parts)
+        )
+
+    @classmethod
+    def _re_partition(cls, regex: ta.Pattern[str], s: str) -> ta.Tuple[str, str, str]:
+        match = regex.search(s)
+        if match:
+            return s[:match.start()], s[slice(*match.span())], s[match.end():]
+        else:
+            return (s, '', '')
+
+    @classmethod
+    def src_to_tokens(cls, src: str) -> ta.List[Token]:
+        tokenize_target = io.StringIO(src)
+        lines = ('', *tokenize_target)
+
+        tokenize_target.seek(0)
+
+        tokens = []
+        last_line = 1
+        last_col = 0
+        end_offset = 0
+
+        gen = tokenize.generate_tokens(tokenize_target.readline)
+        for tok_type, tok_text, (sline, scol), (eline, ecol), line in gen:
+            if sline > last_line:
+                newtok = lines[last_line][last_col:]
+                for lineno in range(last_line + 1, sline):
+                    newtok += lines[lineno]
+                if scol > 0:
+                    newtok += lines[sline][:scol]
+
+                # a multiline unimportant whitespace may contain escaped newlines
+                while cls._ESCAPED_NL_RE.search(newtok):
+                    ws, nl, newtok = cls._re_partition(cls._ESCAPED_NL_RE, newtok)
+                    if ws:
+                        tokens.append(
+                            Token(TokenNames.UNIMPORTANT_WS, ws, last_line, end_offset),
+                        )
+                        end_offset += len(ws.encode())
+                    tokens.append(Token(TokenNames.ESCAPED_NL, nl, last_line, end_offset))
+                    end_offset = 0
+                    last_line += 1
+                if newtok:
+                    tokens.append(Token(TokenNames.UNIMPORTANT_WS, newtok, sline, 0))
+                    end_offset = len(newtok.encode())
+                else:
+                    end_offset = 0
+
+            elif scol > last_col:
+                newtok = line[last_col:scol]
+                tokens.append(Token(TokenNames.UNIMPORTANT_WS, newtok, sline, end_offset))
+                end_offset += len(newtok.encode())
+
+            tok_name = tokenize.tok_name[tok_type]
+
+            if tok_name == 'FSTRING_MIDDLE':  # pragma: >=3.12 cover
+                if '{' in tok_text or '}' in tok_text:
+                    new_tok_text = cls.curly_escape(tok_text)
+                    ecol += len(new_tok_text) - len(tok_text)
+                    tok_text = new_tok_text
+
+            tokens.append(Token(tok_name, tok_text, sline, end_offset))
+            last_line, last_col = eline, ecol
+            if sline != eline:
+                end_offset = len(lines[last_line][:last_col].encode())
+            else:
+                end_offset += len(tok_text.encode())
+
+        return tokens
+
+    @classmethod
+    def parse_string_literal(cls, src: str) -> ta.Tuple[str, str]:
+        """parse a string literal's source into (prefix, string)"""
+        match = check.not_none(cls._STRING_RE.match(src))
+        return match.group(1), match.group(2)
+
+    @classmethod
+    def tokens_to_src(cls, tokens: ta.Iterable[Token]) -> str:
+        return ''.join(tok.src for tok in tokens)
+
+    @classmethod
+    def rfind_string_parts(cls, tokens: ta.Sequence[Token], start: int) -> ta.Tuple[int, ...]:
+        """
+        Find the indicies of the string parts of a (joined) string literal.
+
+        - `i` should start at the end of the string literal
+        - returns `()` (an empty tuple) for things which are not string literals
+        """
+
+        ret = []
+        depth = 0
+        for i in range(start, -1, -1):
+            token = tokens[i]
+            if token.name == 'STRING':
+                ret.append(i)
+            elif token.name in TokenNames.NON_CODING_TOKENS:
+                pass
+            elif token.src == ')':
+                depth += 1
+            elif depth and token.src == '(':
+                depth -= 1
+                # if we closed the paren(s) make sure it was a parenthesized string
+                # and not actually a call
+                if depth == 0:
+                    for j in range(i - 1, -1, -1):
+                        tok = tokens[j]
+                        if tok.name in TokenNames.NON_CODING_TOKENS:
+                            pass
+                        # this was actually a call and not a parenthesized string
+                        elif (
+                                tok.src in {']', ')'} or (
+                                    tok.name == 'NAME' and
+                                    tok.src not in keyword.kwlist
+                                )
+                        ):
+                            return ()
+                        else:
+                            break
+                    break
+            elif depth:  # it looked like a string but wasn't
+                return ()
+            else:
+                break
+        return tuple(reversed(ret))
+
+
+##
+
+
+if __name__ == '__main__':
+    def main(argv: ta.Optional[ta.Sequence[str]] = None) -> int:
+        parser = argparse.ArgumentParser()
+        parser.add_argument('filename')
+        args = parser.parse_args(argv)
+        with open(args.filename) as f:
+            tokens = Tokenization.src_to_tokens(f.read())
+
+        for token in tokens:
+            line, col = str(token.line), str(token.utf8_byte_offset)
+            print(f'{line}:{col} {token.name} {token.src!r}')
+
+        return 0
+
+    raise SystemExit(main())

omdev/{tokens.py → tokens/utils.py}

@@ -1,16 +1,10 @@
 import itertools
 import typing as ta
 
-from omlish import lang
+from .tokenizert import Token
 
 
-if ta.TYPE_CHECKING:
-    import tokenize_rt as trt
-else:
-    trt = lang.proxy_import('tokenize_rt')
-
-
-Tokens: ta.TypeAlias = ta.Sequence['trt.Token']
+Tokens: ta.TypeAlias = ta.Sequence[Token]
 
 
 ##
@@ -25,15 +19,15 @@ WS_NAMES = (
 )
 
 
-def is_ws(tok: 'trt.Token') -> bool:
+def is_ws(tok: Token) -> bool:
     return tok.name in WS_NAMES
 
 
 def ignore_ws(
-        toks: ta.Iterable['trt.Token'],
+        toks: ta.Iterable[Token],
         *,
         keep: ta.Container[str] = (),
-) -> ta.Iterable['trt.Token']:
+) -> ta.Iterable[Token]:
     return (
         t
         for t in toks
@@ -60,7 +54,7 @@ def join_lines(ls: ta.Iterable[Tokens]) -> str:
 
 
 def match_toks(
-        ts: ta.Iterable['trt.Token'],
+        ts: ta.Iterable[Token],
         pat: ta.Sequence[tuple[str | None, str | tuple[str, ...] | None]],
 ) -> bool:
     it = iter(ts)

omdev/tools/mkenv.py

@@ -0,0 +1,131 @@
+"""
+TODO:
+ - detect file extension
+
+==
+
+export $(./python mkenv.py secrets.yml foo_access_token | xargs)
+eval $(om mkenv -e secrets.yml foo_access_token)
+"""
+import argparse
+import json
+import shlex
+import sys
+import typing as ta
+
+from omlish import check
+from omlish.configs.formats import DEFAULT_CONFIG_FILE_LOADER
+from omlish.specs import jmespath
+
+
+##
+
+
+VALUE_TYPES: tuple[type, ...] = (
+    str,
+    int,
+    float,
+    bool,
+)
+
+
+def extract_item(
+        obj: ta.Any,
+        item: str,
+        *,
+        uppercase_keys: bool = False,
+) -> tuple[str, str]:
+    if '=' not in item:
+        k = item
+        v = obj[k]
+        if uppercase_keys:
+            k = k.upper()
+
+    else:
+        k, p = item.split('=')
+        v = jmespath.search(p, obj)
+
+    #
+
+    if isinstance(v, str):
+        s = v
+
+    elif isinstance(v, bool):
+        s = 'true' if v else 'false'
+
+    else:
+        check.isinstance(v, VALUE_TYPES)
+        s = str(v)
+
+    #
+
+    check.equal(s.strip(), s)
+    for c in '\t\n':
+        check.not_in(c, s)
+
+    #
+
+    return (k, s)
+
+
+def extract_items(
+        obj: ta.Any,
+        items: ta.Iterable[str],
+        **kwargs: ta.Any,
+) -> dict[str, str]:
+    return dict(
+        extract_item(obj, item, **kwargs)
+        for item in items
+    )
+
+
+def _main() -> None:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('file')
+    parser.add_argument('-e', '--for-eval', action='store_true')
+    parser.add_argument('-u', '--uppercase', action='store_true')
+    parser.add_argument('item', nargs='*')
+
+    args = parser.parse_args()
+
+    #
+
+    if args.file == '-':
+        obj = json.loads(sys.stdin.read())
+
+    else:
+        data = DEFAULT_CONFIG_FILE_LOADER.load_file(args.file)
+        obj = data.as_map()
+
+    #
+
+    items = extract_items(
+        obj,
+        args.item,
+        uppercase_keys=args.uppercase,
+    )
+
+    #
+
+    if args.for_eval:
+        cmd = ' '.join([
+            'export',
+            *[f'{k}={qv if (qv := shlex.quote(v)) != v else v}' for k, v in items.items()],
+        ])
+        print(cmd)
+
+    else:
+        for k, v in items.items():
+            print(f'{k}={v}')
+
+
+# @omlish-manifest
+_CLI_MODULE = {'$omdev.cli.types.CliModule': {
+    'cmd_name': ['mkenv'],
+    'mod_name': __name__,
+}}
+
+
+if __name__ == '__main__':
+    _main()

omdev/tools/mkrelimp.py

@@ -4,12 +4,10 @@ import logging
 import os.path
 import typing as ta
 
-import tokenize_rt as trt
-
 from omlish.logs import all as logs
 
-from .. import tokens as tks
 from ..cli import CliModule
+from ..tokens import all as tks
 
 
 T = ta.TypeVar('T')
@@ -91,8 +89,8 @@ class Processor:
         ##
 
         new_tks = list(interleave(
-            trt.Token(name='OP', src='.'),
-            [trt.Token(name='NAME', src=p) for p in rel_imp_name_parts],
+            tks.Token(name='OP', src='.'),
+            [tks.Token(name='NAME', src=p) for p in rel_imp_name_parts],
         ))
         out_tks = [
             *pfx,
@@ -111,7 +109,7 @@ class Processor:
         with open(src_file) as f:
             src = f.read()
 
-        ts = trt.src_to_tokens(src)
+        ts = tks.src_to_tokens(src)
         in_ls = tks.split_lines(ts)
         out_ls = [
             self.process_line_tks(

{omdev-0.0.0.dev210.dist-info → omdev-0.0.0.dev212.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev210
+Version: 0.0.0.dev212
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev210
+Requires-Dist: omlish==0.0.0.dev212
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"
@@ -24,7 +24,6 @@ Requires-Dist: mypy~=1.11; extra == "all"
 Requires-Dist: gprof2dot~=2024.6; extra == "all"
 Requires-Dist: prompt-toolkit~=3.0; extra == "all"
 Requires-Dist: segno~=1.6; extra == "all"
-Requires-Dist: tokenize-rt~=6.1; extra == "all"
 Requires-Dist: wheel~=0.44; extra == "all"
 Provides-Extra: black
 Requires-Dist: black~=24.10; extra == "black"
@@ -43,7 +42,5 @@ Provides-Extra: ptk
 Requires-Dist: prompt-toolkit~=3.0; extra == "ptk"
 Provides-Extra: qr
 Requires-Dist: segno~=1.6; extra == "qr"
-Provides-Extra: tokens
-Requires-Dist: tokenize-rt~=6.1; extra == "tokens"
 Provides-Extra: wheel
 Requires-Dist: wheel~=0.44; extra == "wheel"