omlish 0.0.0.dev87__tar.gz → 0.0.0.dev89__tar.gz

{omlish-0.0.0.dev87/omlish.egg-info → omlish-0.0.0.dev89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev87
+Version: 0.0.0.dev89
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev87'
-__revision__ = '116137c4ae1e4aaf7ffb66a2847d4a38d6cce73e'
+__version__ = '0.0.0.dev89'
+__revision__ = '0d88588896911954e58df3f8fcc4b10f17ac59d1'
 
 
 #

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/bootstrap/sys.py

@@ -22,14 +22,14 @@ from .base import SimpleBootstrap
 if ta.TYPE_CHECKING:
     from .. import libc
     from .. import logs
-    from .. import os as osu
     from ..formats import dotenv
+    from ..lite import pidfile
 
 else:
     libc = lang.proxy_import('..libc', __package__)
     logs = lang.proxy_import('..logs', __package__)
-    osu = lang.proxy_import('..os', __package__)
     dotenv = lang.proxy_import('..formats.dotenv', __package__)
+    pidfile = lang.proxy_import('..lite.pidfile', __package__)
 
 
 ##
@@ -317,7 +317,7 @@ class PidfileBootstrap(ContextBootstrap['PidfileBootstrap.Config']):
             yield
             return
 
-        with osu.Pidfile(self._config.path) as pf:
+        with pidfile.Pidfile(self._config.path) as pf:
             pf.write()
             yield
 

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/fnpairs.py

@@ -3,11 +3,10 @@ TODO:
  - objects
   - csv
   - csvloader
-  - cbor
-  - alt json backends
  - wrapped (wait for usecase)
- - streams
+ - streams / incremental
   - fileobj -> fileobj?
+ - swap zstandard for zstd
 
 Compression choice:
  - lzma if-available minimal-space

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/formats/json/cli/cli.py

@@ -1,3 +1,8 @@
+"""
+TODO:
+ - -I/-Ogz, lz4, etc - fnpairs
+ - read from http
+"""
 import argparse
 import codecs
 import contextlib
@@ -11,6 +16,7 @@ import typing as ta
 from .... import check
 from .... import lang
 from .... import term
+from ....lite.io import DelimitingBuffer
 from ..render import JsonRenderer
 from ..stream.build import JsonObjectBuilder
 from ..stream.lex import JsonStreamLexer
@@ -45,7 +51,10 @@ def _main() -> None:
 
     parser.add_argument('--stream', action='store_true')
     parser.add_argument('--stream-build', action='store_true')
-    parser.add_argument('--stream-buffer-size', type=int, default=0x4000)
+
+    parser.add_argument('-l', '--lines', action='store_true')
+
+    parser.add_argument('--read-buffer-size', type=int, default=0x4000)
 
     parser.add_argument('-f', '--format')
 
@@ -58,7 +67,7 @@ def _main() -> None:
 
     parser.add_argument('-c', '--color', action='store_true')
 
-    parser.add_argument('-l', '--less', action='store_true')
+    parser.add_argument('-L', '--less', action='store_true')
 
     args = parser.parse_args()
 
@@ -168,15 +177,14 @@ def _main() -> None:
                 else:
                     renderer = StreamJsonRenderer(
                         out,
-                        StreamJsonRenderer.Options(
-                            **kw,
-                            style=term_color if args.color else None,
-                        ),
+                        style=term_color if args.color else None,
+                        delimiter='\n',
+                        **kw,
                     )
                     build = None
 
                 while True:
-                    buf = os.read(fd, args.stream_buffer_size)
+                    buf = os.read(fd, args.read_buffer_size)
 
                     for s in decoder.decode(buf, not buf):
                         n = 0
@@ -198,7 +206,18 @@ def _main() -> None:
                     if not buf:
                         break
 
-                out.write('\n')
+                if renderer is not None:
+                    out.write('\n')
+
+        elif args.lines:
+            fd = in_file.fileno()
+            db = DelimitingBuffer()
+
+            while buf := os.read(fd, args.read_buffer_size):
+                for chunk in db.feed(buf):
+                    s = check.isinstance(chunk, bytes).decode('utf-8')
+                    v = fmt.load(io.StringIO(s))
+                    print(render_one(v), file=out)
 
         else:
             with io.TextIOWrapper(in_file) as tw:

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/formats/json/render.py

@@ -1,5 +1,4 @@
 import abc
-import dataclasses as dc
 import enum
 import io
 import json
@@ -20,35 +19,32 @@ class AbstractJsonRenderer(lang.Abstract, ta.Generic[I]):
         VALUE = enum.auto()
         KEY = enum.auto()
 
-    @dc.dataclass(frozen=True, kw_only=True)
-    class Options:
-        indent: int | str | None = None
-        separators: tuple[str, str] | None = None
-        sort_keys: bool = False
-        style: ta.Callable[[ta.Any, 'AbstractJsonRenderer.State'], tuple[str, str]] | None = None
-
     def __init__(
             self,
             out: JsonRendererOut,
-            opts: Options = Options(),
+            *,
+            indent: int | str | None = None,
+            separators: tuple[str, str] | None = None,
+            sort_keys: bool = False,
+            style: ta.Callable[[ta.Any, State], tuple[str, str]] | None = None,
     ) -> None:
         super().__init__()
 
         self._out = out
-        self._opts = opts
+        self._sort_keys = sort_keys
+        self._style = style
 
-        separators = opts.separators
-        if isinstance(opts.indent, (str, int)):
-            self._indent = (' ' * opts.indent) if isinstance(opts.indent, int) else opts.indent
+        if isinstance(indent, (str, int)):
+            self._indent = (' ' * indent) if isinstance(indent, int) else indent
             self._endl = '\n'
             if separators is None:
                 separators = (',', ': ')
-        elif opts.indent is None:
+        elif indent is None:
             self._indent = self._endl = ''
             if separators is None:
                 separators = (', ', ': ')
         else:
-            raise TypeError(opts.indent)
+            raise TypeError(indent)
         self._comma, self._colon = separators
 
         self._level = 0
@@ -76,7 +72,7 @@ class AbstractJsonRenderer(lang.Abstract, ta.Generic[I]):
     @classmethod
     def render_str(cls, i: I, **kwargs: ta.Any) -> str:
         out = io.StringIO()
-        cls(out, cls.Options(**kwargs)).render(i)
+        cls(out, **kwargs).render(i)
         return out.getvalue()
 
 
@@ -86,8 +82,8 @@ class JsonRenderer(AbstractJsonRenderer[ta.Any]):
             o: ta.Any,
             state: AbstractJsonRenderer.State = AbstractJsonRenderer.State.VALUE,
     ) -> None:
-        if self._opts.style is not None:
-            pre, post = self._opts.style(o, state)
+        if self._style is not None:
+            pre, post = self._style(o, state)
             self._write(pre)
         else:
             post = None
@@ -102,7 +98,7 @@ class JsonRenderer(AbstractJsonRenderer[ta.Any]):
             self._write('{')
             self._level += 1
             items = list(o.items())
-            if self._opts.sort_keys:
+            if self._sort_keys:
                 items.sort(key=lambda t: t[0])
             for i, (k, v) in enumerate(items):
                 if i:

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/formats/json/stream/lex.py

@@ -94,7 +94,7 @@ CONST_TOKENS: ta.Mapping[str, tuple[TokenKind, str | float | None]] = {
 ##
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass()
 class JsonLexError(Exception):
     message: str
 

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/formats/json/stream/render.py

@@ -3,7 +3,6 @@ import typing as ta
 
 from ..render import AbstractJsonRenderer
 from ..render import JsonRendererOut
-from .build import JsonObjectBuilder
 from .parse import BeginArray
 from .parse import BeginObject
 from .parse import EndArray
@@ -19,23 +18,28 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
     def __init__(
             self,
             out: JsonRendererOut,
-            opts: AbstractJsonRenderer.Options = AbstractJsonRenderer.Options(),
+            *,
+            delimiter: str = '',
+            sort_keys: bool = False,
+            **kwargs: ta.Any,
     ) -> None:
-        if opts.sort_keys:
+        if sort_keys:
             raise TypeError('Not yet implemented')
 
-        super().__init__(out, opts)
+        self._delimiter = delimiter
+
+        super().__init__(out, **kwargs)
 
         self._stack: list[tuple[ta.Literal['OBJECT', 'ARRAY'], int]] = []
-        self._builder: JsonObjectBuilder | None = None
+        self._need_delimit = False
 
     def _render_value(
             self,
             o: ta.Any,
             state: AbstractJsonRenderer.State = AbstractJsonRenderer.State.VALUE,
     ) -> None:
-        if self._opts.style is not None:
-            pre, post = self._opts.style(o, state)
+        if self._style is not None:
+            pre, post = self._style(o, state)
             self._write(pre)
         else:
             post = None
@@ -53,6 +57,10 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
             self._write(post)
 
     def _render(self, e: JsonStreamParserEvent) -> None:
+        if self._need_delimit:
+            self._write(self._delimiter)
+            self._need_delimit = False
+
         if e != EndArray and self._stack and (tt := self._stack[-1])[0] == 'ARRAY':
             if tt[1]:
                 self._write(self._comma)
@@ -64,6 +72,8 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
 
         if e is None or isinstance(e, (str, int, float, bool)):
             self._render_value(e)
+            if not self._stack:
+                self._need_delimit = True
 
         #
 
@@ -92,6 +102,8 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
             if tt[1]:
                 self._write_indent()
             self._write('}')
+            if not self._stack:
+                self._need_delimit = True
 
         #
 
@@ -108,6 +120,8 @@ class StreamJsonRenderer(AbstractJsonRenderer[ta.Iterable[JsonStreamParserEvent]
             if tt[1]:
                 self._write_indent()
             self._write(']')
+            if not self._stack:
+                self._need_delimit = True
 
         #
 

omlish-0.0.0.dev89/omlish/lite/pidfile.py

@@ -0,0 +1,68 @@
+# ruff: noqa: UP007
+import fcntl
+import os
+import signal
+import typing as ta
+
+
+class Pidfile:
+    def __init__(self, path: str) -> None:
+        super().__init__()
+        self._path = path
+
+    _f: ta.TextIO
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}({self._path!r})'
+
+    def __enter__(self) -> 'Pidfile':
+        fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
+        try:
+            os.set_inheritable(fd, True)
+            f = os.fdopen(fd, 'r+')
+        except Exception:
+            try:
+                os.close(fd)
+            except Exception:  # noqa
+                pass
+            raise
+        self._f = f
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._f is not None:
+            self._f.close()
+            del self._f
+
+    def try_lock(self) -> bool:
+        try:
+            fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return True
+        except OSError:
+            return False
+
+    def ensure_locked(self) -> None:
+        if not self.try_lock():
+            raise RuntimeError('Could not get lock')
+
+    def write(self, pid: ta.Optional[int] = None) -> None:
+        self.ensure_locked()
+        if pid is None:
+            pid = os.getpid()
+        self._f.write(f'{pid}\n')
+        self._f.flush()
+
+    def clear(self) -> None:
+        self.ensure_locked()
+        self._f.seek(0)
+        self._f.truncate()
+
+    def read(self) -> int:
+        if self.try_lock():
+            raise RuntimeError('Got lock')
+        self._f.seek(0)
+        return int(self._f.read())
+
+    def kill(self, sig: int = signal.SIGTERM) -> None:
+        pid = self.read()
+        os.kill(pid, sig)  # FIXME: Still racy

omlish-0.0.0.dev89/omlish/os.py

@@ -0,0 +1,59 @@
+import contextlib
+import os
+import resource
+import shutil
+import tempfile
+import typing as ta
+
+
+PAGE_SIZE = resource.getpagesize()
+
+
+def round_to_page_size(sz: int) -> int:
+    sz += PAGE_SIZE - 1
+    return sz - (sz % PAGE_SIZE)
+
+
+@contextlib.contextmanager
+def tmp_dir(
+        root_dir: str | None = None,
+        cleanup: bool = True,
+        **kwargs: ta.Any,
+) -> ta.Iterator[str]:
+    path = tempfile.mkdtemp(dir=root_dir, **kwargs)
+    try:
+        yield path
+    finally:
+        if cleanup:
+            shutil.rmtree(path, ignore_errors=True)
+
+
+@contextlib.contextmanager
+def tmp_file(
+        root_dir: str | None = None,
+        cleanup: bool = True,
+        **kwargs: ta.Any,
+) -> ta.Iterator[tempfile._TemporaryFileWrapper]:  # noqa
+    with tempfile.NamedTemporaryFile(dir=root_dir, delete=False, **kwargs) as f:
+        try:
+            yield f
+        finally:
+            if cleanup:
+                shutil.rmtree(f.name, ignore_errors=True)
+
+
+def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
+    if exist_ok:
+        # First try to bump modification time
+        # Implementation note: GNU touch uses the UTIME_NOW option of the utimensat() / futimens() functions.
+        try:
+            os.utime(self, None)
+        except OSError:
+            pass
+        else:
+            return
+    flags = os.O_CREAT | os.O_WRONLY
+    if not exist_ok:
+        flags |= os.O_EXCL
+    fd = os.open(self, flags, mode)
+    os.close(fd)

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/sql/dbapi.py

@@ -69,6 +69,10 @@ class DbapiDriver:
 
 
 class DbapiDrivers(lang.Namespace):
+
+    ##
+    # sqlite
+
     SQLITE3 = DbapiDriver(
         name='sqlite3',
         dialect=DbapiDialect.SQLITE,
@@ -88,7 +92,8 @@ class DbapiDrivers(lang.Namespace):
         param_style=ParamStyle.QMARK,
     )
 
-    #
+    ##
+    # mysql
 
     PYMYSQL = DbapiDriver(
         name='pymysql',
@@ -112,7 +117,8 @@ class DbapiDrivers(lang.Namespace):
         module_name='MySQLdb',
     )
 
-    #
+    ##
+    # postgres
 
     PG8000 = DbapiDriver(
         name='pg8000',
@@ -132,7 +138,8 @@ class DbapiDrivers(lang.Namespace):
         param_style=ParamStyle.PYFORMAT,
     )
 
-    #
+    ##
+    # snowflake
 
     SNOWFLAKE = DbapiDriver(
         name='snowflake',

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/sql/queries/inserts.py

@@ -18,11 +18,11 @@ from .stmts import Stmt
 
 
 class Values(dc.Frozen, lang.Final):
-    vs: ta.Sequence[Expr]
+    vs: ta.Sequence[Expr] = dc.xfield(coerce=tuple)
 
 
 class Insert(Stmt, lang.Final):
-    columns: ta.Sequence[Ident]
+    columns: ta.Sequence[Ident] = dc.xfield(coerce=tuple)
     into: Relation
     data: Values | Select
 

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish/sql/queries/selects.py

@@ -44,7 +44,7 @@ class SelectBuilder(ExprBuilder, RelationBuilder):
             where: CanExpr | None = None,
     ) -> Select:
         return Select(
-            [self.select_item(i) for i in items],
+            tuple(self.select_item(i) for i in items),
             from_=self.relation(from_) if from_ is not None else None,
             where=self.expr(where) if where is not None else None,
         )

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89/omlish.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omlish
-Version: 0.0.0.dev87
+Version: 0.0.0.dev89
 Summary: omlish
 Author: wrmsr
 License: BSD-3-Clause

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/omlish.egg-info/SOURCES.txt

@@ -306,6 +306,7 @@ omlish/lite/io.py
 omlish/lite/json.py
 omlish/lite/logs.py
 omlish/lite/marshal.py
+omlish/lite/pidfile.py
 omlish/lite/reflect.py
 omlish/lite/runtime.py
 omlish/lite/secrets.py

{omlish-0.0.0.dev87 → omlish-0.0.0.dev89}/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 urls = {source = 'https://github.com/wrmsr/omlish'}
 license = {text = 'BSD-3-Clause'}
 requires-python = '~=3.12'
-version = '0.0.0.dev87'
+version = '0.0.0.dev89'
 classifiers = [
     'License :: OSI Approved :: BSD License',
     'Development Status :: 2 - Pre-Alpha',

omlish-0.0.0.dev87/omlish/os.py

@@ -1,122 +0,0 @@
-import contextlib
-import fcntl
-import os
-import resource
-import shutil
-import signal
-import tempfile
-import typing as ta
-
-
-PAGE_SIZE = resource.getpagesize()
-
-
-def round_to_page_size(sz: int) -> int:
-    sz += PAGE_SIZE - 1
-    return sz - (sz % PAGE_SIZE)
-
-
-@contextlib.contextmanager
-def tmp_dir(
-        root_dir: str | None = None,
-        cleanup: bool = True,
-        **kwargs: ta.Any,
-) -> ta.Iterator[str]:
-    path = tempfile.mkdtemp(dir=root_dir, **kwargs)
-    try:
-        yield path
-    finally:
-        if cleanup:
-            shutil.rmtree(path, ignore_errors=True)
-
-
-@contextlib.contextmanager
-def tmp_file(
-        root_dir: str | None = None,
-        cleanup: bool = True,
-        **kwargs: ta.Any,
-) -> ta.Iterator[tempfile._TemporaryFileWrapper]:  # noqa
-    with tempfile.NamedTemporaryFile(dir=root_dir, delete=False, **kwargs) as f:
-        try:
-            yield f
-        finally:
-            if cleanup:
-                shutil.rmtree(f.name, ignore_errors=True)
-
-
-class Pidfile:
-    def __init__(self, path: str) -> None:
-        super().__init__()
-        self._path = path
-
-    _f: ta.TextIO
-
-    def __repr__(self) -> str:
-        return f'{self.__class__.__name__}({self._path!r})'
-
-    def __enter__(self) -> ta.Self:
-        fd = os.open(self._path, os.O_RDWR | os.O_CREAT, 0o600)
-        try:
-            os.set_inheritable(fd, True)
-            f = os.fdopen(fd, 'r+')
-        except Exception:
-            try:
-                os.close(fd)
-            except Exception:  # noqa
-                pass
-            raise
-        self._f = f
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._f is not None:
-            self._f.close()
-            del self._f
-
-    def try_lock(self) -> bool:
-        try:
-            fcntl.flock(self._f, fcntl.LOCK_EX | fcntl.LOCK_NB)
-            return True
-        except OSError:
-            return False
-
-    def write(self, pid: int | None = None) -> None:
-        if not self.try_lock():
-            raise RuntimeError('Could not get lock')
-        if pid is None:
-            pid = os.getpid()
-        self._f.write(f'{pid}\n')
-        self._f.flush()
-
-    def clear(self) -> None:
-        if not self.try_lock():
-            raise RuntimeError('Could not get lock')
-        self._f.seek(0)
-        self._f.truncate()
-
-    def read(self) -> int:
-        if self.try_lock():
-            raise RuntimeError('Got lock')
-        self._f.seek(0)
-        return int(self._f.read())
-
-    def kill(self, sig: int = signal.SIGTERM) -> None:
-        pid = self.read()
-        os.kill(pid, sig)  # Still racy
-
-
-def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None:
-    if exist_ok:
-        # First try to bump modification time
-        # Implementation note: GNU touch uses the UTIME_NOW option of the utimensat() / futimens() functions.
-        try:
-            os.utime(self, None)
-        except OSError:
-            pass
-        else:
-            return
-    flags = os.O_CREAT | os.O_WRONLY
-    if not exist_ok:
-        flags |= os.O_EXCL
-    fd = os.open(self, flags, mode)
-    os.close(fd)