omlish 0.0.0.dev25__py3-none-any.whl → 0.0.0.dev27__py3-none-any.whl

omlish/__about__.py

@@ -1,5 +1,5 @@
-__version__ = '0.0.0.dev25'
-__revision__ = '123812c0156cbb06af42b6e2ee3b7912d5bf5fc8'
+__version__ = '0.0.0.dev27'
+__revision__ = 'c62e03dc2f032aa532d2c2b9f0d71ff96158cdec'
 
 
 #
@@ -41,7 +41,9 @@ class Project(ProjectBase):
 
         'compression': [
             'lz4 ~= 4.0',
+
             'python-snappy ~= 0.7; python_version < "3.13"',
+
             'zstd ~= 1.5',
         ],
 
@@ -69,6 +71,7 @@ class Project(ProjectBase):
 
         'misc': [
             'jinja2 ~= 3.1',
+
             'wrapt ~= 1.14',
         ],
 
@@ -81,6 +84,7 @@ class Project(ProjectBase):
 
             'pg8000 ~= 1.31',
             # 'psycopg2 ~= 2.9',
+            # 'psycopg ~= 3.2',
 
             'pymysql ~= 1.1',
             # 'mysql-connector-python ~= 9.0',
@@ -122,11 +126,10 @@ class SetuptoolsBase:
         '*': [
             '*.c',
             '*.cc',
+            '*.cu',
             '*.h',
 
-            '*.json',
-
-            '*.sql',
+            '.manifests.json',
 
             'LICENSE',
         ],

omlish/bootstrap/diag.py

@@ -3,6 +3,7 @@ import contextlib
 import dataclasses as dc
 import signal
 import sys
+import threading
 import typing as ta
 
 from .. import check
@@ -17,6 +18,7 @@ if ta.TYPE_CHECKING:
     import pstats
 
     from ..diag import pycharm as diagpc
+    from ..diag import replserver as diagrs
     from ..diag import threads as diagt
 
 else:
@@ -24,6 +26,7 @@ else:
     pstats = lang.proxy_import('pstats')
 
     diagpc = lang.proxy_import('..diag.pycharm', __package__)
+    diagrs = lang.proxy_import('..diag.replserver', __package__)
     diagt = lang.proxy_import('..diag.threads', __package__)
 
 
@@ -175,3 +178,25 @@ class PycharmBootstrap(SimpleBootstrap['PycharmBootstrap.Config']):
                 self._config.debug_port,
                 version=self._config.version,
             )
+
+
+##
+
+
+class ReplServerBootstrap(ContextBootstrap['ReplServerBootstrap.Config']):
+    @dc.dataclass(frozen=True)
+    class Config(Bootstrap.Config):
+        path: str | None = None
+
+    @contextlib.contextmanager
+    def enter(self) -> ta.Iterator[None]:
+        if self._config.path is None:
+            return
+
+        with diagrs.ReplServer(diagrs.ReplServer.Config(
+            path=self._config.path,
+        )) as rs:
+            thread = threading.Thread(target=rs.run, name='replserver')
+            thread.start()
+
+            yield

omlish/bootstrap/harness.py

@@ -6,6 +6,7 @@ TODO:
  - multiprocess profiling - afterfork, suffix with pid
 
 TODO diag:
+ - tracemalloc
  - yappi
  - stackscope
  - https://github.com/pythonspeed/filprofiler
@@ -13,10 +14,10 @@ TODO diag:
  - https://pypi.org/project/memory-profiler/
  - https://pypi.org/project/Pympler/
 
+
 TODO new items:
- - pydevd connect-back
  - debugging / pdb
- - repl server
+  - https://github.com/inducer/pudb
  - packaging fixups
  - daemonize ( https://github.com/thesharp/daemonize/blob/master/daemonize.py )
 """

omlish/check.py

@@ -332,7 +332,7 @@ def single(obj: ta.Iterable[T], message: Message = None) -> T:
     return value
 
 
-def optional_single(obj: ta.Iterable[T], message: Message = None) -> T | None:
+def opt_single(obj: ta.Iterable[T], message: Message = None) -> T | None:
     it = iter(obj)
     try:
         value = next(it)

omlish/collections/__init__.py

@@ -91,6 +91,7 @@ from .unmodifiable import (  # noqa
 )
 
 from .utils import (  # noqa
+    PartitionResult,
     all_equal,
     all_not_equal,
     indexes,

omlish/collections/utils.py

@@ -39,7 +39,12 @@ def toposort(data: ta.Mapping[T, ta.AbstractSet[T]]) -> ta.Iterator[set[T]]:
 ##
 
 
-def partition(items: ta.Iterable[T], pred: ta.Callable[[T], bool]) -> tuple[list[T], list[T]]:
+class PartitionResult(ta.NamedTuple, ta.Generic[T]):
+    t: list[T]
+    f: list[T]
+
+
+def partition(items: ta.Iterable[T], pred: ta.Callable[[T], bool]) -> PartitionResult[T]:
     t: list[T] = []
     f: list[T] = []
     for e in items:
@@ -47,7 +52,7 @@ def partition(items: ta.Iterable[T], pred: ta.Callable[[T], bool]) -> tuple[list
             t.append(e)
         else:
             f.append(e)
-    return t, f
+    return PartitionResult(t, f)
 
 
 def unique(

omlish/dataclasses/__init__.py

@@ -59,8 +59,8 @@ globals()['make_dataclass'] = xmake_dataclass
 
 
 from .impl.exceptions import (  # noqa
-    CheckError,
-    FieldCheckError,
+    FieldValidationError,
+    ValidationError,
 )
 
 from .impl.metaclass import (  # noqa
@@ -76,8 +76,8 @@ from .impl.metadata import (  # noqa
     UserMetadata,
     metadata,
 
-    Check,
-    check,
+    Validate,
+    validate,
 
     Init,
     init,

omlish/dataclasses/impl/api.py

@@ -30,8 +30,9 @@ def field(  # noqa
         metadata=None,
         kw_only=MISSING,
 
-        coerce: ta.Callable[[ta.Any], ta.Any] | None = None,
-        check: ta.Callable[[ta.Any], bool] | None = None,
+        derive: ta.Callable[..., ta.Any] | None = None,
+        coerce: bool | ta.Callable[[ta.Any], ta.Any] | None = None,
+        validate: ta.Callable[[ta.Any], bool] | None = None,
         check_type: bool | None = None,
         override: bool = False,
         repr_fn: ta.Callable[[ta.Any], str | None] | None = None,
@@ -40,8 +41,9 @@ def field(  # noqa
         raise ValueError('cannot specify both default and default_factory')
 
     fx = FieldExtras(
+        derive=derive,
         coerce=coerce,
-        check=check,
+        validate=validate,
         check_type=check_type,
         override=override,
         repr_fn=repr_fn,

omlish/dataclasses/impl/exceptions.py

@@ -1,8 +1,8 @@
-class CheckError(Exception):
+class ValidationError(Exception):
     pass
 
 
-class FieldCheckError(CheckError):
+class FieldValidationError(ValidationError):
     def __init__(self, field: str) -> None:
         super().__init__(field)
         self.field = field

omlish/dataclasses/impl/fields.py

@@ -175,15 +175,18 @@ def field_init(
     else:
         pass
 
+    if fx.derive is not None:
+        raise NotImplementedError
+
     if fx.coerce is not None:
         cn = f'__dataclass_coerce__{f.name}__'
         locals[cn] = fx.coerce
         lines.append(f'{value} = {cn}({value})')
 
-    if fx.check is not None:
-        cn = f'__dataclass_check__{f.name}__'
-        locals[cn] = fx.check
-        lines.append(f'if not {cn}({value}): raise __dataclass_FieldCheckError__({f.name})')
+    if fx.validate is not None:
+        cn = f'__dataclass_validate__{f.name}__'
+        locals[cn] = fx.validate
+        lines.append(f'if not {cn}({value}): raise __dataclass_FieldValidationError__({f.name})')
 
     if fx.check_type:
         cn = f'__dataclass_check_type__{f.name}__'

omlish/dataclasses/impl/init.py

@@ -3,16 +3,16 @@ import inspect
 import typing as ta
 
 from ... import lang
-from .exceptions import CheckError
-from .exceptions import FieldCheckError
+from .exceptions import FieldValidationError
+from .exceptions import ValidationError
 from .fields import field_init
 from .fields import field_type
 from .fields import has_default
 from .internals import HAS_DEFAULT_FACTORY
 from .internals import POST_INIT_NAME
 from .internals import FieldType
-from .metadata import Check
 from .metadata import Init
+from .metadata import Validate
 from .processing import Processor
 from .reflect import ClassInfo
 from .utils import Namespace
@@ -100,8 +100,8 @@ class InitBuilder:
             '__dataclass_builtins_object__': object,
             '__dataclass_builtins_isinstance__': isinstance,
             '__dataclass_builtins_TypeError__': TypeError,
-            '__dataclass_CheckError__': CheckError,
-            '__dataclass_FieldCheckError__': FieldCheckError,
+            '__dataclass_ValidationError__': ValidationError,
+            '__dataclass_FieldValidationError__': FieldValidationError,
         })
 
         body_lines: list[str] = []
@@ -121,14 +121,14 @@ class InitBuilder:
             params_str = ','.join(f.name for f in ifs.all if field_type(f) is FieldType.INIT)
             body_lines.append(f'{self._self_name}.{POST_INIT_NAME}({params_str})')
 
-        for i, fn in enumerate(self._info.merged_metadata.get(Check, [])):
+        for i, fn in enumerate(self._info.merged_metadata.get(Validate, [])):
             if isinstance(fn, staticmethod):
                 fn = fn.__func__
-            cn = f'__dataclass_check_{i}__'
+            cn = f'__dataclass_validate_{i}__'
             locals[cn] = fn
             csig = inspect.signature(fn)
             cas = ', '.join(p.name for p in csig.parameters.values())
-            body_lines.append(f'if not {cn}({cas}): raise __dataclass_CheckError__')
+            body_lines.append(f'if not {cn}({cas}): raise __dataclass_ValidationError__')
 
         inits = self._info.merged_metadata.get(Init, [])
         mro_dct = lang.build_mro_dict(self._info.cls)

omlish/dataclasses/impl/metadata.py

@@ -54,12 +54,12 @@ def metadata(cls_dct, *args) -> None:
 
 
 @_class_merged
-class Check(lang.Marker):
+class Validate(lang.Marker):
     pass
 
 
-def check(fn: ta.Callable[..., bool] | staticmethod) -> None:
-    _append_cls_md(Check, fn)
+def validate(fn: ta.Callable[..., bool] | staticmethod) -> None:
+    _append_cls_md(Validate, fn)
 
 
 ##

omlish/dataclasses/impl/params.py

@@ -40,10 +40,11 @@ from .metadata import METADATA_ATTR
 ##
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass(frozen=True, kw_only=True)
 class FieldExtras(lang.Final):
+    derive: ta.Callable[..., ta.Any] | None = None
     coerce: bool | ta.Callable[[ta.Any], ta.Any] | None = None
-    check: ta.Callable[[ta.Any], bool] | None = None
+    validate: ta.Callable[[ta.Any], bool] | None = None
     check_type: bool | None = None
     override: bool = False
     repr_fn: ta.Callable[[ta.Any], str | None] | None = None
@@ -79,7 +80,7 @@ def get_params(obj: ta.Any) -> Params:
 ##
 
 
-@dc.dataclass(frozen=True)
+@dc.dataclass(frozen=True, kw_only=True)
 class ParamsExtras(lang.Final):
     reorder: bool = False
     cache_hash: bool = False

omlish/diag/replserver/server.py

@@ -1,8 +1,13 @@
 """
+FIXME:
+ - lol shutdown deadlocks
+ - whole thing is just gross is this its own thread or what
+
 TODO:
  - !!! ANYIO !!!
  - optional paramiko ssh-server
  - optional ipython embed
+ - https://github.com/python/cpython/tree/56470004e58911b146c016fc9fec4461b8f69454/Lib/_pyrepl
 
 lookit:
  - https://github.com/vxgmichel/aioconsole/blob/e55f4b0601da3b3a40a88c965526d35ab38b5841/aioconsole/server.py
@@ -38,7 +43,7 @@ class ReplServer:
     @dc.dataclass(frozen=True)
     class Config:
         path: str
-        file_mode: int | None = None
+        file_mode: int | None = 0o660
         poll_interval: float = 0.5
         exit_timeout: float = 10.0
 
@@ -80,7 +85,17 @@ class ReplServer:
 
         self._socket = sock.socket(sock.AF_UNIX, sock.SOCK_STREAM)
         self._socket.settimeout(self._config.poll_interval)
-        self._socket.bind(self._config.path)
+
+        if self._config.file_mode is not None:
+            prev_umask = os.umask(~self._config.file_mode)
+        else:
+            prev_umask = None
+        try:
+            self._socket.bind(self._config.path)
+        finally:
+            if prev_umask is not None:
+                os.umask(prev_umask)
+
         with contextlib.closing(self._socket):
             self._socket.listen(1)
 

omlish/docker.py

@@ -7,12 +7,6 @@ TODO:
   - https://stackoverflow.com/questions/55386202/how-can-i-use-the-docker-registry-api-to-pull-information-about-a-container-get
   - https://ops.tips/blog/inspecting-docker-image-without-pull/
   
-repo=library/nginx
-token=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" | jq -r '.token')
-curl -H "Authorization: Bearer $token" -s "https://registry-1.docker.io/v2/${repo}/tags/list" | jq
-api="application/vnd.docker.distribution.manifest.v2+json"
-apil="application/vnd.docker.distribution.manifest.list.v2+json"
-curl -H "Accept: ${api}" -H "Accept: ${apil}" -H "Authorization: Bearer $token" -s "https://registry-1.docker.io/v2/${repo}/manifests/latest" | jq .
 """  # noqa
 import datetime
 import os
@@ -21,6 +15,7 @@ import shlex
 import subprocess
 import sys
 import typing as ta
+import urllib.request
 
 from . import check
 from . import dataclasses as dc
@@ -191,3 +186,76 @@ DOCKER_HOST_PLATFORM_KEY = 'DOCKER_HOST_PLATFORM'
 
 def get_docker_host_platform() -> str | None:
     return os.environ.get(DOCKER_HOST_PLATFORM_KEY)
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class HubRepoInfo:
+    repo: str
+    tags: ta.Mapping[str, ta.Any]
+    latest_manifests: ta.Mapping[str, ta.Any]
+
+
+def get_hub_repo_info(
+        repo: str,
+        *,
+        auth_url: str = 'https://auth.docker.io/',
+        api_url: str = 'https://registry-1.docker.io/v2/',
+) -> HubRepoInfo:
+    """
+    https://stackoverflow.com/a/39376254
+
+    ==
+
+    repo=library/nginx
+    token=$(
+        curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull" \
+        | jq -r '.token' \
+    )
+    curl -H "Authorization: Bearer $token" -s "https://registry-1.docker.io/v2/${repo}/tags/list" | jq
+    curl \
+        -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+        -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
+        -H "Authorization: Bearer $token" \
+        -s "https://registry-1.docker.io/v2/${repo}/manifests/latest" \
+    | jq .
+    """
+
+    auth_url = auth_url.rstrip('/')
+    api_url = api_url.rstrip('/')
+
+    #
+
+    def req_json(url: str, **kwargs: ta.Any) -> ta.Any:
+        with urllib.request.urlopen(urllib.request.Request(url, **kwargs)) as resp:  # noqa
+            return json.loads(resp.read().decode('utf-8'))
+
+    #
+
+    token_dct = req_json(f'{auth_url}/token?service=registry.docker.io&scope=repository:{repo}:pull')
+    token = token_dct['token']
+
+    req_hdrs = {'Authorization': f'Bearer {token}'}
+
+    #
+
+    tags_dct = req_json(
+        f'{api_url}/{repo}/tags/list',
+        headers=req_hdrs,
+    )
+
+    latest_mani_dct = req_json(
+        f'{api_url}/{repo}/manifests/latest',
+        headers={
+            **req_hdrs,
+            'Accept': 'application/vnd.docker.distribution.manifest.v2+json',
+        },
+    )
+
+    return HubRepoInfo(
+        repo,
+        tags_dct,
+        latest_mani_dct,
+    )

omlish/formats/yaml.py

@@ -5,6 +5,7 @@ TODO:
  - goal: perfect rewrites (comments, whitespace)
   - or at least comments
  - rename 'objects'? codecs/serde interplay still unresolved
+ - look ma, a monad
 """
 import datetime
 import types
@@ -26,6 +27,9 @@ else:
 T = ta.TypeVar('T')
 
 
+##
+
+
 @dc.dataclass(frozen=True)
 class NodeWrapped(lang.Final, ta.Generic[T]):
     value: T
@@ -124,6 +128,9 @@ class NodeWrappingConstructorMixin:
         return self.__construct_yaml_pairs(node, super().construct_yaml_pairs)  # type: ignore  # noqa
 
 
+##
+
+
 class _cached_class_property:  # noqa
     def __init__(self, fn):
         super().__init__()
@@ -204,6 +211,9 @@ class WrappedLoaders(lang.Namespace):
         return cls.CUnsafe(*args, **kwargs)
 
 
+##
+
+
 def load(stream, Loader):  # noqa
     with lang.disposing(Loader(stream)) as loader:
         return loader.get_single_data()

omlish/graphs/dot/__init__.py

@@ -1,19 +1,31 @@
-from .items import Attrs  # noqa
-from .items import Cell  # noqa
-from .items import Edge  # noqa
-from .items import Graph  # noqa
-from .items import Id  # noqa
-from .items import Item  # noqa
-from .items import Node  # noqa
-from .items import Raw  # noqa
-from .items import RawStmt  # noqa
-from .items import Row  # noqa
-from .items import Stmt  # noqa
-from .items import Table  # noqa
-from .items import Text  # noqa
-from .rendering import Renderer  # noqa
-from .rendering import open_dot  # noqa
-from .rendering import render  # noqa
-from .utils import Color  # noqa
-from .utils import escape  # noqa
-from .utils import gen_rainbow  # noqa
+from .items import (  # noqa
+    Attrs,
+    Cell,
+    Edge,
+    Graph,
+    Id,
+    Item,
+    Node,
+    Raw,
+    RawStmt,
+    Row,
+    Stmt,
+    Table,
+    Text,
+)
+
+from .make import (  # noqa
+    make_simple,
+)
+
+from .rendering import (  # noqa
+    Renderer,
+    open_dot,
+    render,
+)
+
+from .utils import (  # noqa
+    Color,
+    escape,
+    gen_rainbow,
+)

omlish/graphs/dot/make.py

@@ -0,0 +1,16 @@
+import typing as ta
+
+from ... import lang
+from .items import Edge
+from .items import Graph
+from .items import Node
+
+
+T = ta.TypeVar('T')
+
+
+def make_simple(graph: ta.Mapping[T, ta.Iterable[T]]) -> Graph:
+    return Graph([
+        *[Node(n) for n in {*graph, *lang.flatten(graph.values())}],
+        *[Edge(k, v) for k, vs in graph.items() for v in vs],
+    ])

omlish/graphs/dot/rendering.py

@@ -120,6 +120,7 @@ def open_dot(
         *,
         timeout_s: float = 1.,
         sleep_s: float = 0.,
+        delete: bool = False,
 ) -> None:
     stdout, _ = subprocess.Popen(
         ['dot', '-Tpdf'],
@@ -132,16 +133,16 @@ def open_dot(
 
     with tempfile.NamedTemporaryFile(
         suffix='.pdf',
-        delete=True,
+        delete=delete,
     ) as pdf:
         pdf.file.write(stdout)
         pdf.file.flush()
 
-        _, _ = subprocess.Popen(
-            ['open', pdf.name],
-        ).communicate(
-            timeout=timeout_s,
-        )
+    _, _ = subprocess.Popen(
+        ['open', pdf.name],
+    ).communicate(
+        timeout=timeout_s,
+    )
 
-        if sleep_s > 0.:
-            time.sleep(sleep_s)
+    if sleep_s > 0.:
+        time.sleep(sleep_s)

omlish/http/cookies.py

@@ -29,6 +29,7 @@ import typing as ta
 import urllib.parse
 
 from .. import collections as col
+from .. import lang
 from .dates import http_date
 
 
@@ -140,7 +141,7 @@ def dump_cookie(
         if not isinstance(expires, str):
             expires = http_date(expires)
     elif max_age is not None and sync_expires:
-        expires = http_date(datetime.datetime.now(tz=datetime.UTC).timestamp() + max_age)
+        expires = http_date(lang.utcnow().timestamp() + max_age)
 
     if samesite is not None:
         samesite = samesite.title()

omlish/inject/impl/scopes.py

@@ -1,6 +1,8 @@
 """
 TODO:
  - ContextVar ('context')
+ - greenlet?
+ - dynamic? https://github.com/wrmsr/iceworm/blob/2f6b4d5e9d237ef9665f7d57cfa6ce328efa0757/iceworm/utils/inject.py#L44
 """
 import abc
 import contextlib

omlish/inject/keys.py

@@ -18,7 +18,7 @@ class Key(lang.Final, ta.Generic[T]):
     tag: ta.Any = dc.xfield(
         default=None,
         kw_only=True,
-        check=lambda o: not isinstance(o, Tag),
+        validate=lambda o: not isinstance(o, Tag),
         repr_fn=dc.opt_repr,
     )