omdev 0.0.0.dev210__py3-none-any.whl → 0.0.0.dev212__py3-none-any.whl

omdev/.manifests.json

@@ -326,11 +326,25 @@
       }
     }
   },
+  {
+    "module": ".tools.mkenv",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/mkenv.py",
+    "line": 123,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": [
+          "mkenv"
+        ],
+        "mod_name": "omdev.tools.mkenv"
+      }
+    }
+  },
   {
     "module": ".tools.mkrelimp",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/mkrelimp.py",
-    "line": 148,
+    "line": 146,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "py/mkrelimp",

omdev/__about__.py

@@ -45,10 +45,6 @@ class Project(ProjectBase):
             'segno ~= 1.6',
         ],
 
-        'tokens': [
-            'tokenize-rt ~= 6.1',
-        ],
-
         'wheel': [
             'wheel ~= 0.44',
         ],

omdev/amalg/gen.py

@@ -8,11 +8,10 @@ from omlish import collections as col
 from omlish import lang
 from omlish.lite.runtime import LITE_REQUIRED_PYTHON_VERSION
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .srcfiles import SrcFile
 from .srcfiles import make_src_file
 from .strip import strip_main_lines
-from .types import Tokens
 from .typing import Typing
 
 
@@ -71,7 +70,7 @@ class AmalgGenerator:
         return self._src_files()[self._main_path]
 
     @cached.function
-    def _header_lines(self) -> list[Tokens]:
+    def _header_lines(self) -> list[str]:
         header_lines = []
 
         if self._main_file().header_lines:

omdev/amalg/imports.py

@@ -4,8 +4,7 @@ import typing as ta
 
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -22,11 +21,11 @@ class Import:
 
     mod_path: str | None
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def make_import(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
         mounts: ta.Mapping[str, str],
@@ -90,7 +89,7 @@ def make_import(
         as_=as_,
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         mod_path=mod_path,
 

omdev/amalg/manifests.py

@@ -1,17 +1,14 @@
 import ast
 
-import tokenize_rt as trt
-
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
 
 
-def is_manifest_comment(line: Tokens) -> bool:
+def is_manifest_comment(line: tks.Tokens) -> bool:
     if not line:
         return False
 
@@ -22,10 +19,10 @@ def is_manifest_comment(line: Tokens) -> bool:
 
 
 def comment_out_manifest_comment(
-        line: Tokens,
-        cls: list[Tokens],
+        line: tks.Tokens,
+        cls: list[tks.Tokens],
         i: int,
-) -> tuple[list[Tokens], int]:
+) -> tuple[list[tks.Tokens], int]:
     mls = [line]
     while True:
         mls.append(cls[i])
@@ -41,8 +38,8 @@ def comment_out_manifest_comment(
         check.isinstance(check.single(mmod.body), ast.Assign)
         break
 
-    out: list[Tokens] = [
-        [trt.Token('COMMENT', '# ' + tks.join_toks(ml))]
+    out: list[tks.Tokens] = [
+        [tks.Token('COMMENT', '# ' + tks.join_toks(ml))]
         for ml in mls
     ]
 

omdev/amalg/resources.py

@@ -4,10 +4,7 @@ import itertools
 import os.path
 import typing as ta
 
-import tokenize_rt as trt
-
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -19,7 +16,7 @@ class RootLevelResourcesRead(ta.NamedTuple):
     resource: str
 
 
-def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
+def is_root_level_resources_read(lts: tks.Tokens) -> RootLevelResourcesRead | None:
     wts = list(tks.ignore_ws(lts, keep=['INDENT']))
 
     if not tks.match_toks(wts, [
@@ -47,36 +44,36 @@ def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
 def build_resource_lines(
         rsrc: RootLevelResourcesRead,
         path: str,
-) -> list[Tokens]:
+) -> list[tks.Tokens]:
     rf = os.path.join(os.path.dirname(path), rsrc.resource)
 
     if rsrc.kind == 'binary':
         with open(rf, 'rb') as bf:
             rb = bf.read()  # noqa
 
-        out: list[Tokens] = [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='NAME', src='base64'),
-            trt.Token(name='OP', src='.'),
-            trt.Token(name='NAME', src='b64decode'),
-            trt.Token(name='OP', src='('),
-            trt.Token(name='NL', src='\n'),
+        out: list[tks.Tokens] = [[
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='NAME', src='base64'),
+            tks.Token(name='OP', src='.'),
+            tks.Token(name='NAME', src='b64decode'),
+            tks.Token(name='OP', src='('),
+            tks.Token(name='NL', src='\n'),
         ]]
 
         rb64 = base64.b64encode(rb).decode('ascii')
         for chunk in itertools.batched(rb64, 96):
             out.append([
-                trt.Token(name='UNIMPORTANT_WS', src='    '),
-                trt.Token(name='STRING', src=f"'{''.join(chunk)}'"),
-                trt.Token(name='NL', src='\n'),
+                tks.Token(name='UNIMPORTANT_WS', src='    '),
+                tks.Token(name='STRING', src=f"'{''.join(chunk)}'"),
+                tks.Token(name='NL', src='\n'),
             ])
 
         out.append([
-            trt.Token(name='OP', src=')'),
-            trt.Token(name='NEWLINE', src='\n'),
+            tks.Token(name='OP', src=')'),
+            tks.Token(name='NEWLINE', src='\n'),
         ])
 
         return out
@@ -87,12 +84,12 @@ def build_resource_lines(
         rt = rt.replace('\\', '\\\\')  # Escape backslashes
         rt = rt.replace('"""', r'\"\"\"')
         return [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
-            trt.Token(name='NEWLINE', src=''),
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
+            tks.Token(name='NEWLINE', src=''),
         ]]
 
     else:

omdev/amalg/srcfiles.py

@@ -1,12 +1,10 @@
 import dataclasses as dc
 import typing as ta
 
-import tokenize_rt as trt
-
 from omlish import collections as col
 from omlish import lang
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .imports import Import
 from .imports import make_import
 from .manifests import comment_out_manifest_comment
@@ -15,7 +13,6 @@ from .resources import build_resource_lines
 from .resources import is_root_level_resources_read
 from .strip import split_header_lines
 from .strip import strip_header_lines
-from .types import Tokens
 from .typing import Typing
 from .typing import is_root_level_if_type_checking_block
 from .typing import make_typing
@@ -30,13 +27,13 @@ class SrcFile:
     path: str
 
     src: str = dc.field(repr=False)
-    tokens: Tokens = dc.field(repr=False)
-    lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    tokens: tks.Tokens = dc.field(repr=False)
+    lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
-    header_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    header_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
     imports: ta.Sequence[Import] = dc.field(repr=False)
     typings: ta.Sequence[Typing] = dc.field(repr=False)
-    content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    content_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
     ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
 
@@ -51,7 +48,7 @@ def make_src_file(
     with open(path) as f:
         src = f.read().strip()
 
-    tokens = trt.src_to_tokens(src)
+    tokens = tks.src_to_tokens(src)
     lines = tks.split_lines(tokens)
 
     header_lines, cls = split_header_lines(lines)
@@ -61,7 +58,7 @@ def make_src_file(
 
     imps: list[Import] = []
     tys: list[Typing] = []
-    ctls: list[Tokens] = []
+    ctls: list[tks.Tokens] = []
 
     has_binary_resources = False
 

omdev/amalg/strip.py

@@ -2,8 +2,7 @@ import re
 import typing as ta
 
 from .. import magic
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -12,7 +11,7 @@ from .types import Tokens
 HEADER_NAMES = (*tks.WS_NAMES, 'COMMENT', 'STRING')
 
 
-def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[Tokens]]:
+def split_header_lines(lines: ta.Iterable[tks.Tokens]) -> tuple[list[tks.Tokens], list[tks.Tokens]]:
     ws = []
     nws = []
     for line in (it := iter(lines)):
@@ -31,7 +30,7 @@ def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[T
 IF_MAIN_PAT = re.compile(r'if\s+__name__\s+==\s+[\'"]__main__[\'"]\s*:')
 
 
-def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_main_lines(cls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     out = []
 
     for l in (it := iter(cls)):
@@ -59,7 +58,7 @@ STRIPPED_HEADER_PAT = magic.compile_magic_style_pat(
 )
 
 
-def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_header_lines(hls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     if hls and tks.join_toks(hls[0]).startswith('#!'):
         hls = hls[1:]
     out = []

omdev/amalg/types.py

@@ -1,6 +1,6 @@
 import typing as ta
 
-from .. import tokens as tks
+from ..tokens import all as tks
 
 
 Tokens: ta.TypeAlias = tks.Tokens

omdev/amalg/typing.py

@@ -1,7 +1,8 @@
 import dataclasses as dc
 
-from .. import tokens as tks
-from .types import Tokens
+from omlish import check
+
+from ..tokens import all as tks
 
 
 ##
@@ -18,11 +19,11 @@ class Typing:
     src_path: str
     line: int
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def _is_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         exclude_newtypes: bool = False,
 ) -> bool:
@@ -48,7 +49,7 @@ def _is_typing(
 
 
 def make_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
 ) -> Typing | None:
@@ -63,7 +64,7 @@ def make_typing(
         src=tks.join_toks(lts),
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         toks=lts,
     )
@@ -72,7 +73,7 @@ def make_typing(
 ##
 
 
-def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
+def is_root_level_if_type_checking_block(lts: tks.Tokens) -> bool:
     return tks.match_toks(tks.ignore_ws(lts, keep=['INDENT']), [
         ('NAME', 'if'),
         ('NAME', 'ta'),
@@ -83,7 +84,7 @@ def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
 
 
 def skip_root_level_if_type_checking_block(
-        cls: list[Tokens],
+        cls: list[tks.Tokens],
         i: int,
 ) -> int:
     def skip_block():

omdev/ci/cache.py

@@ -2,21 +2,24 @@
 # @omlish-lite
 import abc
 import os.path
+import shlex
 import shutil
 import typing as ta
 
+from .shell import ShellCmd
 
-#
+
+##
 
 
 @abc.abstractmethod
 class FileCache(abc.ABC):
     @abc.abstractmethod
-    def get_file(self, name: str) -> ta.Optional[str]:
+    def get_file(self, key: str) -> ta.Optional[str]:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def put_file(self, name: str) -> ta.Optional[str]:
+    def put_file(self, key: str, file_path: str) -> ta.Optional[str]:
         raise NotImplementedError
 
 
@@ -29,13 +32,137 @@ class DirectoryFileCache(FileCache):
 
         self._dir = dir
 
-    def get_file(self, name: str) -> ta.Optional[str]:
-        file_path = os.path.join(self._dir, name)
-        if not os.path.exists(file_path):
+    #
+
+    def get_cache_file_path(
+            self,
+            key: str,
+            *,
+            make_dirs: bool = False,
+    ) -> str:
+        if make_dirs:
+            os.makedirs(self._dir, exist_ok=True)
+        return os.path.join(self._dir, key)
+
+    def format_incomplete_file(self, f: str) -> str:
+        return os.path.join(os.path.dirname(f), f'_{os.path.basename(f)}.incomplete')
+
+    #
+
+    def get_file(self, key: str) -> ta.Optional[str]:
+        cache_file_path = self.get_cache_file_path(key)
+        if not os.path.exists(cache_file_path):
             return None
-        return file_path
+        return cache_file_path
 
-    def put_file(self, file_path: str) -> None:
-        os.makedirs(self._dir, exist_ok=True)
-        cache_file_path = os.path.join(self._dir, os.path.basename(file_path))
+    def put_file(self, key: str, file_path: str) -> None:
+        cache_file_path = self.get_cache_file_path(key, make_dirs=True)
         shutil.copyfile(file_path, cache_file_path)
+
+
+##
+
+
+class ShellCache(abc.ABC):
+    @abc.abstractmethod
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        raise NotImplementedError
+
+    class PutFileCmdContext(abc.ABC):
+        def __init__(self) -> None:
+            super().__init__()
+
+            self._state: ta.Literal['open', 'committed', 'aborted'] = 'open'
+
+        @property
+        def state(self) -> ta.Literal['open', 'committed', 'aborted']:
+            return self._state
+
+        #
+
+        @property
+        @abc.abstractmethod
+        def cmd(self) -> ShellCmd:
+            raise NotImplementedError
+
+        #
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            if exc_val is None:
+                self.commit()
+            else:
+                self.abort()
+
+        #
+
+        @abc.abstractmethod
+        def _commit(self) -> None:
+            raise NotImplementedError
+
+        def commit(self) -> None:
+            if self._state == 'committed':
+                return
+            elif self._state == 'open':
+                self._commit()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+        #
+
+        @abc.abstractmethod
+        def _abort(self) -> None:
+            raise NotImplementedError
+
+        def abort(self) -> None:
+            if self._state == 'aborted':
+                return
+            elif self._state == 'open':
+                self._abort()
+                self._state = 'committed'
+            else:
+                raise RuntimeError(self._state)
+
+    @abc.abstractmethod
+    def put_file_cmd(self, key: str) -> PutFileCmdContext:
+        raise NotImplementedError
+
+
+#
+
+
+class DirectoryShellCache(ShellCache):
+    def __init__(self, dfc: DirectoryFileCache) -> None:
+        super().__init__()
+
+        self._dfc = dfc
+
+    def get_file_cmd(self, key: str) -> ta.Optional[ShellCmd]:
+        f = self._dfc.get_file(key)
+        if f is None:
+            return None
+        return ShellCmd(f'cat {shlex.quote(f)}')
+
+    class _PutFileCmdContext(ShellCache.PutFileCmdContext):  # noqa
+        def __init__(self, tf: str, f: str) -> None:
+            super().__init__()
+
+            self._tf = tf
+            self._f = f
+
+        @property
+        def cmd(self) -> ShellCmd:
+            return ShellCmd(f'cat > {shlex.quote(self._tf)}')
+
+        def _commit(self) -> None:
+            os.replace(self._tf, self._f)
+
+        def _abort(self) -> None:
+            os.unlink(self._tf)
+
+    def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
+        f = self._dfc.get_cache_file_path(key, make_dirs=True)
+        return self._PutFileCmdContext(self._dfc.format_incomplete_file(f), f)