omdev 0.0.0.dev211__tar.gz → 0.0.0.dev213__tar.gz

{omdev-0.0.0.dev211/omdev.egg-info → omdev-0.0.0.dev213}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: omdev
-Version: 0.0.0.dev211
+Version: 0.0.0.dev213
 Summary: omdev
 Author: wrmsr
 License: BSD-3-Clause
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Operating System :: POSIX
 Requires-Python: >=3.12
 License-File: LICENSE
-Requires-Dist: omlish==0.0.0.dev211
+Requires-Dist: omlish==0.0.0.dev213
 Provides-Extra: all
 Requires-Dist: black~=24.10; extra == "all"
 Requires-Dist: pycparser~=2.22; extra == "all"
@@ -24,7 +24,6 @@ Requires-Dist: mypy~=1.11; extra == "all"
 Requires-Dist: gprof2dot~=2024.6; extra == "all"
 Requires-Dist: prompt-toolkit~=3.0; extra == "all"
 Requires-Dist: segno~=1.6; extra == "all"
-Requires-Dist: tokenize-rt~=6.1; extra == "all"
 Requires-Dist: wheel~=0.44; extra == "all"
 Provides-Extra: black
 Requires-Dist: black~=24.10; extra == "black"
@@ -43,7 +42,5 @@ Provides-Extra: ptk
 Requires-Dist: prompt-toolkit~=3.0; extra == "ptk"
 Provides-Extra: qr
 Requires-Dist: segno~=1.6; extra == "qr"
-Provides-Extra: tokens
-Requires-Dist: tokenize-rt~=6.1; extra == "tokens"
 Provides-Extra: wheel
 Requires-Dist: wheel~=0.44; extra == "wheel"

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/.manifests.json

@@ -326,11 +326,25 @@
       }
     }
   },
+  {
+    "module": ".tools.mkenv",
+    "attr": "_CLI_MODULE",
+    "file": "omdev/tools/mkenv.py",
+    "line": 123,
+    "value": {
+      "$.cli.types.CliModule": {
+        "cmd_name": [
+          "mkenv"
+        ],
+        "mod_name": "omdev.tools.mkenv"
+      }
+    }
+  },
   {
     "module": ".tools.mkrelimp",
     "attr": "_CLI_MODULE",
     "file": "omdev/tools/mkrelimp.py",
-    "line": 148,
+    "line": 146,
     "value": {
       "$.cli.types.CliModule": {
         "cmd_name": "py/mkrelimp",

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/__about__.py

@@ -45,10 +45,6 @@ class Project(ProjectBase):
             'segno ~= 1.6',
         ],
 
-        'tokens': [
-            'tokenize-rt ~= 6.1',
-        ],
-
         'wheel': [
             'wheel ~= 0.44',
         ],

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/gen.py

@@ -8,11 +8,10 @@ from omlish import collections as col
 from omlish import lang
 from omlish.lite.runtime import LITE_REQUIRED_PYTHON_VERSION
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .srcfiles import SrcFile
 from .srcfiles import make_src_file
 from .strip import strip_main_lines
-from .types import Tokens
 from .typing import Typing
 
 
@@ -71,7 +70,7 @@ class AmalgGenerator:
         return self._src_files()[self._main_path]
 
     @cached.function
-    def _header_lines(self) -> list[Tokens]:
+    def _header_lines(self) -> list[str]:
         header_lines = []
 
         if self._main_file().header_lines:

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/imports.py

@@ -4,8 +4,7 @@ import typing as ta
 
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -22,11 +21,11 @@ class Import:
 
     mod_path: str | None
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def make_import(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
         mounts: ta.Mapping[str, str],
@@ -90,7 +89,7 @@ def make_import(
         as_=as_,
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         mod_path=mod_path,
 

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/manifests.py

@@ -1,17 +1,14 @@
 import ast
 
-import tokenize_rt as trt
-
 from omlish import check
 
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
 
 
-def is_manifest_comment(line: Tokens) -> bool:
+def is_manifest_comment(line: tks.Tokens) -> bool:
     if not line:
         return False
 
@@ -22,10 +19,10 @@ def is_manifest_comment(line: Tokens) -> bool:
 
 
 def comment_out_manifest_comment(
-        line: Tokens,
-        cls: list[Tokens],
+        line: tks.Tokens,
+        cls: list[tks.Tokens],
         i: int,
-) -> tuple[list[Tokens], int]:
+) -> tuple[list[tks.Tokens], int]:
     mls = [line]
     while True:
         mls.append(cls[i])
@@ -41,8 +38,8 @@ def comment_out_manifest_comment(
         check.isinstance(check.single(mmod.body), ast.Assign)
         break
 
-    out: list[Tokens] = [
-        [trt.Token('COMMENT', '# ' + tks.join_toks(ml))]
+    out: list[tks.Tokens] = [
+        [tks.Token('COMMENT', '# ' + tks.join_toks(ml))]
         for ml in mls
     ]
 

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/resources.py

@@ -4,10 +4,7 @@ import itertools
 import os.path
 import typing as ta
 
-import tokenize_rt as trt
-
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -19,7 +16,7 @@ class RootLevelResourcesRead(ta.NamedTuple):
     resource: str
 
 
-def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
+def is_root_level_resources_read(lts: tks.Tokens) -> RootLevelResourcesRead | None:
     wts = list(tks.ignore_ws(lts, keep=['INDENT']))
 
     if not tks.match_toks(wts, [
@@ -47,36 +44,36 @@ def is_root_level_resources_read(lts: Tokens) -> RootLevelResourcesRead | None:
 def build_resource_lines(
         rsrc: RootLevelResourcesRead,
         path: str,
-) -> list[Tokens]:
+) -> list[tks.Tokens]:
     rf = os.path.join(os.path.dirname(path), rsrc.resource)
 
     if rsrc.kind == 'binary':
         with open(rf, 'rb') as bf:
             rb = bf.read()  # noqa
 
-        out: list[Tokens] = [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='NAME', src='base64'),
-            trt.Token(name='OP', src='.'),
-            trt.Token(name='NAME', src='b64decode'),
-            trt.Token(name='OP', src='('),
-            trt.Token(name='NL', src='\n'),
+        out: list[tks.Tokens] = [[
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='NAME', src='base64'),
+            tks.Token(name='OP', src='.'),
+            tks.Token(name='NAME', src='b64decode'),
+            tks.Token(name='OP', src='('),
+            tks.Token(name='NL', src='\n'),
         ]]
 
         rb64 = base64.b64encode(rb).decode('ascii')
         for chunk in itertools.batched(rb64, 96):
             out.append([
-                trt.Token(name='UNIMPORTANT_WS', src='    '),
-                trt.Token(name='STRING', src=f"'{''.join(chunk)}'"),
-                trt.Token(name='NL', src='\n'),
+                tks.Token(name='UNIMPORTANT_WS', src='    '),
+                tks.Token(name='STRING', src=f"'{''.join(chunk)}'"),
+                tks.Token(name='NL', src='\n'),
             ])
 
         out.append([
-            trt.Token(name='OP', src=')'),
-            trt.Token(name='NEWLINE', src='\n'),
+            tks.Token(name='OP', src=')'),
+            tks.Token(name='NEWLINE', src='\n'),
         ])
 
         return out
@@ -87,12 +84,12 @@ def build_resource_lines(
         rt = rt.replace('\\', '\\\\')  # Escape backslashes
         rt = rt.replace('"""', r'\"\"\"')
         return [[
-            trt.Token(name='NAME', src=rsrc.variable),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='OP', src='='),
-            trt.Token(name='UNIMPORTANT_WS', src=' '),
-            trt.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
-            trt.Token(name='NEWLINE', src=''),
+            tks.Token(name='NAME', src=rsrc.variable),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='OP', src='='),
+            tks.Token(name='UNIMPORTANT_WS', src=' '),
+            tks.Token(name='STRING', src=f'"""\\\n{rt}"""  # noqa\n'),
+            tks.Token(name='NEWLINE', src=''),
         ]]
 
     else:

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/srcfiles.py

@@ -1,12 +1,10 @@
 import dataclasses as dc
 import typing as ta
 
-import tokenize_rt as trt
-
 from omlish import collections as col
 from omlish import lang
 
-from .. import tokens as tks
+from ..tokens import all as tks
 from .imports import Import
 from .imports import make_import
 from .manifests import comment_out_manifest_comment
@@ -15,7 +13,6 @@ from .resources import build_resource_lines
 from .resources import is_root_level_resources_read
 from .strip import split_header_lines
 from .strip import strip_header_lines
-from .types import Tokens
 from .typing import Typing
 from .typing import is_root_level_if_type_checking_block
 from .typing import make_typing
@@ -30,13 +27,13 @@ class SrcFile:
     path: str
 
     src: str = dc.field(repr=False)
-    tokens: Tokens = dc.field(repr=False)
-    lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    tokens: tks.Tokens = dc.field(repr=False)
+    lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
-    header_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    header_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
     imports: ta.Sequence[Import] = dc.field(repr=False)
     typings: ta.Sequence[Typing] = dc.field(repr=False)
-    content_lines: ta.Sequence[Tokens] = dc.field(repr=False)
+    content_lines: ta.Sequence[tks.Tokens] = dc.field(repr=False)
 
     ruff_noqa: ta.AbstractSet[str] = dc.field(repr=False)
 
@@ -51,7 +48,7 @@ def make_src_file(
     with open(path) as f:
         src = f.read().strip()
 
-    tokens = trt.src_to_tokens(src)
+    tokens = tks.src_to_tokens(src)
     lines = tks.split_lines(tokens)
 
     header_lines, cls = split_header_lines(lines)
@@ -61,7 +58,7 @@ def make_src_file(
 
     imps: list[Import] = []
     tys: list[Typing] = []
-    ctls: list[Tokens] = []
+    ctls: list[tks.Tokens] = []
 
     has_binary_resources = False
 

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/strip.py

@@ -2,8 +2,7 @@ import re
 import typing as ta
 
 from .. import magic
-from .. import tokens as tks
-from .types import Tokens
+from ..tokens import all as tks
 
 
 ##
@@ -12,7 +11,7 @@ from .types import Tokens
 HEADER_NAMES = (*tks.WS_NAMES, 'COMMENT', 'STRING')
 
 
-def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[Tokens]]:
+def split_header_lines(lines: ta.Iterable[tks.Tokens]) -> tuple[list[tks.Tokens], list[tks.Tokens]]:
     ws = []
     nws = []
     for line in (it := iter(lines)):
@@ -31,7 +30,7 @@ def split_header_lines(lines: ta.Iterable[Tokens]) -> tuple[list[Tokens], list[T
 IF_MAIN_PAT = re.compile(r'if\s+__name__\s+==\s+[\'"]__main__[\'"]\s*:')
 
 
-def strip_main_lines(cls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_main_lines(cls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     out = []
 
     for l in (it := iter(cls)):
@@ -59,7 +58,7 @@ STRIPPED_HEADER_PAT = magic.compile_magic_style_pat(
 )
 
 
-def strip_header_lines(hls: ta.Sequence[Tokens]) -> list[Tokens]:
+def strip_header_lines(hls: ta.Sequence[tks.Tokens]) -> list[tks.Tokens]:
     if hls and tks.join_toks(hls[0]).startswith('#!'):
         hls = hls[1:]
     out = []

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/types.py

@@ -1,6 +1,6 @@
 import typing as ta
 
-from .. import tokens as tks
+from ..tokens import all as tks
 
 
 Tokens: ta.TypeAlias = tks.Tokens

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/amalg/typing.py

@@ -1,7 +1,8 @@
 import dataclasses as dc
 
-from .. import tokens as tks
-from .types import Tokens
+from omlish import check
+
+from ..tokens import all as tks
 
 
 ##
@@ -18,11 +19,11 @@ class Typing:
     src_path: str
     line: int
 
-    toks: Tokens = dc.field(repr=False)
+    toks: tks.Tokens = dc.field(repr=False)
 
 
 def _is_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         exclude_newtypes: bool = False,
 ) -> bool:
@@ -48,7 +49,7 @@ def _is_typing(
 
 
 def make_typing(
-        lts: Tokens,
+        lts: tks.Tokens,
         *,
         src_path: str,
 ) -> Typing | None:
@@ -63,7 +64,7 @@ def make_typing(
         src=tks.join_toks(lts),
 
         src_path=src_path,
-        line=ft.line,
+        line=check.not_none(ft.line),
 
         toks=lts,
     )
@@ -72,7 +73,7 @@ def make_typing(
 ##
 
 
-def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
+def is_root_level_if_type_checking_block(lts: tks.Tokens) -> bool:
     return tks.match_toks(tks.ignore_ws(lts, keep=['INDENT']), [
         ('NAME', 'if'),
         ('NAME', 'ta'),
@@ -83,7 +84,7 @@ def is_root_level_if_type_checking_block(lts: Tokens) -> bool:
 
 
 def skip_root_level_if_type_checking_block(
-        cls: list[Tokens],
+        cls: list[tks.Tokens],
         i: int,
 ) -> int:
     def skip_block():

omdev-0.0.0.dev213/omdev/cc/cdeps.py

@@ -0,0 +1,71 @@
+import dataclasses as dc
+import tomllib
+import typing as ta
+
+from omlish import cached
+from omlish import lang
+from omlish import marshal as msh
+from omlish.configs import all as configs
+
+
+@dc.dataclass(frozen=True)
+class Cdep:
+    @dc.dataclass(frozen=True)
+    class Git:
+        url: str
+        rev: str
+
+        subtrees: ta.Sequence[str] | None = None
+
+    git: Git
+
+    #
+
+    sources: ta.Sequence[str] | None = None
+    include: ta.Sequence[str] | None = None
+
+    #
+
+    @dc.dataclass(frozen=True)
+    class Cmake:
+        @dc.dataclass(frozen=True)
+        class FetchContent:
+            url: str | None = None
+
+            @dc.dataclass(frozen=True)
+            class Git:
+                repository: str | None = None
+                tag: str | None = None
+
+            git: Git | None = None
+
+        fetch_content: FetchContent | None = None
+
+    cmake: Cmake | None = None
+
+
+def process_marshaled_cdep(obj: ta.Any) -> ta.Any:
+    obj = configs.processing.matched_rewrite(
+        lambda s: s if isinstance(s, str) else ''.join(s),
+        obj,
+        ('sources', None),
+        ('include', None),
+    )
+
+    return obj
+
+
+@cached.function
+def load_cdeps() -> ta.Mapping[str, Cdep]:
+    src = lang.get_relative_resources(globals=globals())['cdeps.toml'].read_text()
+    dct = tomllib.loads(src)
+
+    dct = {
+        **dct,
+        'deps': {
+            k: process_marshaled_cdep(v)
+            for k, v in dct.get('deps', {}).items()
+        },
+    }
+
+    return msh.unmarshal(dct.get('deps', {}), ta.Mapping[str, Cdep])  # type: ignore

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/cc/cdeps.toml

@@ -1,3 +1,18 @@
+[deps.httplib]
+include = ['.']
+
+[deps.httplib.git]
+url = 'https://github.com/yhirose/cpp-httplib'
+rev = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+subtrees = ['httplib.h']
+
+[deps.httplib.cmake.fetch_content.git]
+repository = 'https://github.com/yhirose/cpp-httplib.git'
+tag = 'a7bc00e3307fecdb4d67545e93be7b88cfb1e186'
+
+
+#
+
 [deps.json]
 include = ['single_include']
 
@@ -6,8 +21,9 @@ url = 'https://github.com/nlohmann/json'
 rev = '9cca280a4d0ccf0c08f47a99aa71d1b0e52f8d03'
 subtrees = ['single_include']
 
-[deps.json.cmake]
-fetch_content_url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+[deps.json.cmake.fetch_content]
+url = 'https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz'
+
 
 #
 
@@ -19,6 +35,7 @@ url = 'https://github.com/pybind/pybind11'
 rev = '945e251a6ce7273058b36214d94415e9c9530b8e'
 subtrees = ['include']
 
+
 #
 
 [deps.ut]

{omdev-0.0.0.dev211 → omdev-0.0.0.dev213}/omdev/cc/cli.py

@@ -18,6 +18,8 @@ Freestanding options:
 TODO:
  - cext interop
  - gen cmake
+ - fix CFLAGS/CCFLAGS/CPPFLAGS/CXXFLAGS
+ - jit-gen cmake mode? multi-src builds
 """
 import os
 import shlex
@@ -35,6 +37,7 @@ from .. import magic
 from ..cache import data as dcache
 from .cdeps import Cdep
 from .cdeps import load_cdeps
+from .cdeps import process_marshaled_cdep
 
 
 class Cli(ap.Cli):
@@ -64,16 +67,22 @@ class Cli(ap.Cli):
             if src_magic.key == '@omlish-cdeps':
                 for dep in check.isinstance(src_magic.prepared, ta.Sequence):
                     if isinstance(dep, ta.Mapping):
-                        dep = msh.unmarshal(dep, Cdep)  # type: ignore
+                        dep = process_marshaled_cdep(dep)
+                        dep = msh.unmarshal(dep, Cdep)
                     else:
                         dep = load_cdeps()[check.isinstance(dep, str)]
 
+                    if dep.sources:
+                        # TODO
+                        raise NotImplementedError
+
                     dep_spec = dcache.GitSpec(
                         url=dep.git.url,
                         rev=dep.git.rev,
                         subtrees=dep.git.subtrees,
                     )
                     dep_dir = dcache.default().get(dep_spec)
+
                     for dep_inc in dep.include or []:
                         inc_dir = os.path.join(dep_dir, dep_inc)
                         check.state(os.path.isdir(inc_dir))
@@ -96,6 +105,9 @@ class Cli(ap.Cli):
         if cflags := os.environ.get('CFLAGS'):
             sh_parts.append(cflags)  # Explicitly shell-unquoted
 
+        if ldflags := os.environ.get('LDFLAGS'):
+            sh_parts.append(ldflags)  # Explicitly shell-unquoted
+
         sh_parts.extend([
             '-std=c++20',
             shlex.quote(os.path.abspath(src_file)),