omlish 0.0.0.dev4__py3-none-any.whl → 0.0.0.dev6__py3-none-any.whl

omlish/fnpairs.py

@@ -18,31 +18,36 @@ import typing as ta
 
 from . import lang
 
+
 if ta.TYPE_CHECKING:
     import bz2 as _bz2
-    import cloudpickle as _cloudpickle
     import gzip as _gzip
     import json as _json
-    import lz4.frame as _lz4_frame
     import lzma as _lzma
     import pickle as _pickle
-    import snappy as _snappy
     import struct as _struct
     import tomllib as _tomllib
+
+    import cloudpickle as _cloudpickle
+    import json5 as _json5
+    import lz4.frame as _lz4_frame
+    import snappy as _snappy
     import yaml as _yaml
     import zstd as _zstd
 
 else:
     _bz2 = lang.proxy_import('bz2')
-    _cloudpickle = lang.proxy_import('cloudpickle')
     _gzip = lang.proxy_import('gzip')
     _json = lang.proxy_import('json')
-    _lz4_frame = lang.proxy_import('lz4.frame')
     _lzma = lang.proxy_import('lzma')
     _pickle = lang.proxy_import('pickle')
-    _snappy = lang.proxy_import('snappy')
     _struct = lang.proxy_import('struct')
     _tomllib = lang.proxy_import('tomllib')
+
+    _cloudpickle = lang.proxy_import('cloudpickle')
+    _json5 = lang.proxy_import('json5')
+    _lz4_frame = lang.proxy_import('lz4.frame')
+    _snappy = lang.proxy_import('snappy')
     _yaml = lang.proxy_import('yaml')
     _zstd = lang.proxy_import('zstd')
 
@@ -177,13 +182,13 @@ UTF8 = text('utf-8')
 
 
 @dc.dataclass(frozen=True)
-class Optional(FnPair[ta.Optional[F], ta.Optional[T]]):
+class Optional(FnPair[F | None, T | None]):
     fp: FnPair[F, T]
 
-    def forward(self, f: ta.Optional[F]) -> ta.Optional[T]:
+    def forward(self, f: F | None) -> T | None:
         return None if f is None else self.fp.forward(f)
 
-    def backward(self, t: ta.Optional[T]) -> ta.Optional[F]:
+    def backward(self, t: T | None) -> F | None:
         return None if t is None else self.fp.backward(t)
 
 
@@ -305,15 +310,20 @@ class Struct(FnPair[tuple, bytes]):
 ##
 
 
-class Object(FnPair[ta.Any, T], lang.Abstract):  # noqa
+Object: ta.TypeAlias = FnPair[ta.Any, T]
+ObjectStr: ta.TypeAlias = Object[str]
+ObjectBytes: ta.TypeAlias = Object[bytes]
+
+
+class Object_(FnPair[ta.Any, T], lang.Abstract):  # noqa
     pass
 
 
-class ObjectStr(Object[str], lang.Abstract):  # noqa
+class ObjectStr_(Object_[str], lang.Abstract):  # noqa
     pass
 
 
-class ObjectBytes(Object[bytes], lang.Abstract):  # noqa
+class ObjectBytes_(Object_[bytes], lang.Abstract):  # noqa
     pass
 
 
@@ -322,7 +332,7 @@ class ObjectBytes(Object[bytes], lang.Abstract):  # noqa
 
 @_register_extension('pkl')
 @dc.dataclass(frozen=True)
-class Pickle(ObjectBytes):
+class Pickle(ObjectBytes_):
     protocol: int | None = None
 
     def forward(self, f: ta.Any) -> bytes:
@@ -334,7 +344,7 @@ class Pickle(ObjectBytes):
 
 @_register_extension('json')
 @dc.dataclass(frozen=True)
-class Json(ObjectStr):
+class Json(ObjectStr_):
     indent: int | str | None = dc.field(default=None, kw_only=True)
     separators: tuple[str, str] | None = dc.field(default=None, kw_only=True)
 
@@ -360,7 +370,7 @@ class JsonLines(FnPair[ta.Sequence[ta.Any], str]):
 
 
 @_register_extension('toml')
-class Toml(ObjectStr):
+class Toml(ObjectStr_):
     def forward(self, f: ta.Any) -> str:
         raise NotImplementedError
 
@@ -373,7 +383,7 @@ class Toml(ObjectStr):
 
 @_register_extension('cpkl')
 @dc.dataclass(frozen=True)
-class Cloudpickle(ObjectBytes):
+class Cloudpickle(ObjectBytes_):
     protocol: int | None = None
 
     def forward(self, f: ta.Any) -> bytes:
@@ -383,8 +393,17 @@ class Cloudpickle(ObjectBytes):
         return _cloudpickle.loads(t)
 
 
+@_register_extension('json5')
+class Json5(ObjectStr_):
+    def forward(self, f: ta.Any) -> str:
+        return _json5.dumps(f)
+
+    def backward(self, t: str) -> ta.Any:
+        return _json5.loads(t)
+
+
 @_register_extension('yml', 'yaml')
-class Yaml(ObjectStr):
+class Yaml(ObjectStr_):
     def forward(self, f: ta.Any) -> str:
         return _yaml.dump(f)
 
@@ -392,7 +411,7 @@ class Yaml(ObjectStr):
         return _yaml.safe_load(t)
 
 
-class YamlUnsafe(ObjectStr):
+class YamlUnsafe(ObjectStr_):
     def forward(self, f: ta.Any) -> str:
         return _yaml.dump(f)
 

omlish/graphs/dags.py

@@ -0,0 +1,113 @@
+"""
+TODO:
+ - parser?
+ - js? viz.js, d3, visjs
+ - cycle detection
+ - networkx adapter
+ - https://docs.python.org/3.9/library/graphlib.html#module-graphlib
+"""
+import typing as ta
+
+from .. import check
+from .. import lang
+
+
+K = ta.TypeVar('K')
+V = ta.TypeVar('V')
+T = ta.TypeVar('T')
+U = ta.TypeVar('U')
+
+
+def traverse_links(data: ta.Mapping[T, ta.Iterable[T]], keys: ta.Iterable[T]) -> set[T]:
+    keys = set(keys)
+    todo = set(keys)
+    seen: set[T] = set()
+    while todo:
+        key = todo.pop()
+        seen.add(key)
+        cur = data.get(key, [])
+        todo.update(set(cur) - seen)
+    return seen - keys
+
+
+def invert_set_map(src: ta.Mapping[K, ta.Iterable[V]]) -> dict[V, set[K]]:
+    dst: dict[V, set[K]] = {}
+    for l, rs in src.items():
+        for r in rs:
+            try:
+                s = dst[r]
+            except KeyError:
+                s = dst[r] = set()
+            s.add(l)
+    return dst
+
+
+def invert_symmetric_set_map(src: ta.Mapping[T, ta.Iterable[T]]) -> dict[T, set[T]]:
+    dst: dict[T, set[T]] = {l: set() for l in src}
+    for l, rs in src.items():
+        for r in rs:
+            dst[r].add(l)
+    return dst
+
+
+class Dag(ta.Generic[T]):
+
+    def __init__(self, input_its_by_outputs: ta.Mapping[T, ta.Iterable[T]]) -> None:
+        super().__init__()
+
+        self._input_sets_by_output = {u: set(d) for u, d in input_its_by_outputs.items()}
+
+    @property
+    def input_sets_by_output(self) -> ta.Mapping[T, ta.AbstractSet[T]]:
+        return self._input_sets_by_output
+
+    @lang.cached_property
+    def output_sets_by_input(self) -> ta.Mapping[T, ta.AbstractSet[T]]:
+        return invert_symmetric_set_map(self._input_sets_by_output)
+
+    def subdag(self, *args, **kwargs) -> 'Subdag[T]':
+        return Subdag(self, *args, **kwargs)
+
+
+class Subdag(ta.Generic[U]):
+
+    def __init__(
+            self,
+            dag: 'Dag[U]',
+            targets: ta.Iterable[U],
+            *,
+            ignored: ta.Iterable[U] | None = None,
+    ) -> None:
+        super().__init__()
+
+        self._dag: Dag[U] = check.isinstance(dag, Dag)  # type: ignore
+        self._targets = set(targets)
+        self._ignored = set(ignored or []) - self._targets
+
+    @property
+    def dag(self) -> 'Dag[U]':
+        return self._dag
+
+    @property
+    def targets(self) -> ta.AbstractSet[U]:
+        return self._targets
+
+    @property
+    def ignored(self) -> ta.AbstractSet[U]:
+        return self._ignored
+
+    @lang.cached_property
+    def inputs(self) -> ta.AbstractSet[U]:
+        return traverse_links(self.dag.input_sets_by_output, self.targets) - self.ignored
+
+    @lang.cached_property
+    def outputs(self) -> ta.AbstractSet[U]:
+        return traverse_links(self.dag.output_sets_by_input, self.targets) - self.ignored
+
+    @lang.cached_property
+    def output_inputs(self) -> ta.AbstractSet[U]:
+        return traverse_links(self.dag.input_sets_by_output, self.outputs) - self.ignored
+
+    @lang.cached_property
+    def all(self) -> ta.AbstractSet[U]:
+        return self.targets | self.inputs | self.outputs | self.output_inputs

omlish/graphs/domination.py

@@ -0,0 +1,268 @@
+import abc
+import contextlib
+import typing as ta
+
+from .. import check
+from .. import lang
+
+
+V = ta.TypeVar('V')
+MK = ta.TypeVar('MK')
+MV = ta.TypeVar('MV')
+SetMap = ta.Mapping[MK, ta.AbstractSet[MV]]
+
+
+class DirectedGraph(ta.Generic[V], lang.Abstract):
+
+    @abc.abstractmethod
+    def get_successors(self, vertex: V) -> ta.Collection[V]:
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def yield_depth_first(self, root: V) -> ta.Iterator[V]:
+        raise NotImplementedError
+
+
+class ListDictDirectedGraph(DirectedGraph[V]):
+
+    def __init__(self, items: ta.Iterable[tuple[V, ta.Iterable[V]]]) -> None:
+        super().__init__()
+
+        lst_dct: dict[V, list[V]] = {}
+        all_children = set()
+        for parent, children in items:
+            check.not_in(parent, lst_dct)
+            lst = []
+            seen = set()
+            for child in children:
+                if child not in seen:
+                    seen.add(child)
+                    lst.append(child)
+                    all_children.add(child)
+            lst_dct[parent] = lst
+        check.empty(all_children - set(lst_dct))
+        self._lst_dct = lst_dct
+
+    def get_successors(self, vertex: V) -> ta.Collection[V]:
+        return self._lst_dct[vertex]
+
+    def yield_depth_first(self, root: V) -> ta.Iterator[V]:
+        stack: list[V] = [root]
+        seen: set[V] = set()
+        while stack:
+            cur = stack.pop()
+            yield cur
+            for child in self._lst_dct[cur]:
+                if child not in seen:
+                    seen.add(child)
+                    stack.append(child)
+
+
+class DominatorTree(ta.Generic[V]):
+
+    def __init__(self, graph: DirectedGraph[V], root: V) -> None:
+        super().__init__()
+
+        self._graph = check.not_none(graph)
+        self._root = check.not_none(root)
+        check.not_none(self._graph.get_successors(root))
+        self._dfs = _Dfs(graph, root)
+
+    @lang.cached_property
+    def immediate_dominators(self) -> ta.Mapping[V, V]:
+        return _ImmediateDominanceComputer(self._dfs).immediate_dominators
+
+    @lang.cached_property
+    def dominator_tree(self) -> SetMap[V, V]:
+        tree: dict[V, set[V]] = {}
+        for node, dom in self.immediate_dominators.items():
+            tree.setdefault(dom, set()).add(node)
+        return tree
+
+    @lang.cached_property
+    def deep_dominated(self) -> SetMap[V, V]:
+        seen: set[V] = set()
+        ret: dict[V, set[V]] = {}
+
+        def rec(node: V) -> ta.Collection[V]:
+            check.not_in(node, seen)
+            seen.add(node)
+            # FIXME: pyrsistent
+            st = set()
+            for child in self.dominator_tree.get(node, []):
+                st.add(child)
+                st.update(rec(child))
+            if st:
+                ret[node] = st
+            return st
+
+        rec(self._root)
+        return ret
+
+    @lang.cached_property
+    def dominance_frontiers(self) -> SetMap[V, V]:
+        dominance_frontiers: dict[V, set[V]] = {}
+
+        for x in self.reverse_topological_traversal:
+            dfx = dominance_frontiers.setdefault(x, set())
+
+            for y in self._graph.get_successors(x):
+                if self.immediate_dominators[y] != x:
+                    dfx.add(y)
+
+            for z in self.dominator_tree.get(x, []):
+                for y in dominance_frontiers.get(z, []):
+                    if self.immediate_dominators[y] != x:
+                        dfx.add(y)
+
+        return {k: v for k, v in dominance_frontiers.items() if v}
+
+    @lang.cached_property
+    def topological_traversal(self) -> list[V]:
+        # FIXME: LinkedList
+        lst: list[V] = []
+
+        for node in self._dfs.vertex:
+            try:
+                idx = lst.index(self.immediate_dominators[node])
+            except (KeyError, ValueError):
+                lst.append(node)
+            else:
+                lst.insert(idx + 1, node)
+
+        return lst
+
+    @lang.cached_property
+    def reverse_topological_traversal(self) -> list[V]:
+        return list(reversed(self.topological_traversal))
+
+
+class _Dfs(ta.Generic[V]):
+
+    def __init__(self, graph: DirectedGraph[V], root: V) -> None:
+        super().__init__()
+
+        semi: dict[V, int] = {}
+        vertex: list[V] = []
+        parent: dict[V, V] = {}
+        pred: dict[V, set[V]] = {}
+        label: dict[V, V] = {}
+
+        for node in graph.yield_depth_first(root):
+            if node not in semi:
+                vertex.append(node)
+
+                check.not_in(node, semi)
+                semi[node] = len(semi)
+                check.not_in(node, label)
+                label[node] = node
+
+                for child in graph.get_successors(node):
+                    pred.setdefault(child, set()).add(node)
+                    if child not in semi:
+                        check.not_in(child, parent)
+                        parent[child] = node
+
+        self._semi = semi
+        self._vertex = vertex
+        self._parent = parent
+        self._pred = pred
+        self._label = label
+
+    @property
+    def semi(self) -> dict[V, int]:
+        return self._semi
+
+    @property
+    def vertex(self) -> list[V]:
+        return self._vertex
+
+    @property
+    def parent(self) -> dict[V, V]:
+        return self._parent
+
+    @property
+    def pred(self) -> dict[V, set[V]]:
+        return self._pred
+
+    @property
+    def label(self) -> dict[V, V]:
+        return self._label
+
+
+class _ImmediateDominanceComputer(ta.Generic[V]):
+
+    def __init__(self, dfs: _Dfs[V]) -> None:
+        super().__init__()
+
+        self._dfs: _Dfs[V] = check.isinstance(dfs, _Dfs)  # type: ignore
+
+        self._ancestor: dict[V, V] = {}
+        self._semi = dict(self._dfs.semi)
+        self._label = dict(self._dfs.label)
+
+    @lang.cached_property
+    def immediate_dominators(self) -> ta.Mapping[V, V]:
+        idom: dict[V, V] = {}
+        bucket: dict[V, set[V]] = {}
+
+        last_semi_number = len(self._semi) - 1
+
+        for i in range(last_semi_number, 0, -1):
+            w = self._dfs.vertex[i]
+            p = self._dfs.parent[w]
+
+            semidominator = self._semi[w]
+            for v in self._dfs.pred.get(w, []):
+                semidominator = min(semidominator, self._semi[self._eval(v)])
+
+            self._semi[w] = semidominator
+            bucket.setdefault(self._dfs.vertex[semidominator], set()).add(w)
+
+            self._ancestor[w] = p
+
+            for v in bucket.get(p, []):
+                u = self._eval(v)
+
+                if self._semi[u] < self._semi[v]:
+                    idom[v] = u
+                else:
+                    idom[v] = p
+
+            with contextlib.suppress(KeyError):
+                del bucket[p]
+
+        for i in range(1, last_semi_number + 1):
+            w = self._dfs.vertex[i]
+
+            if idom[w] != self._dfs.vertex[self._semi[w]]:
+                idom[w] = idom[idom[w]]
+
+        return idom
+
+    def _eval(self, v: V) -> V:
+        self._compress(v)
+        return self._label[v]
+
+    def _compress(self, v: V) -> None:
+        worklist: list[V] = [v]
+
+        a = self._ancestor.get(v)
+
+        while a in self._ancestor:
+            worklist.append(a)
+            a = self._ancestor[a]
+
+        ancestor = worklist.pop()
+        least_semi = self._semi[self._label[ancestor]]
+
+        while worklist:
+            descendent = worklist.pop()
+            current_semi = self._semi[self._label[descendent]]
+
+            if current_semi > least_semi:
+                self._label[descendent] = self._label[ancestor]
+            else:
+                least_semi = current_semi
+
+            ancestor = descendent

omlish/graphs/trees.py

@@ -79,7 +79,7 @@ class BasicTreeAnalysis(ta.Generic[NodeT]):
         nodes: list[NodeT] = []
         node_set: ta.MutableSet[NodeT] = self._set_fac()
         children_by_node: ta.MutableMapping[NodeT | None, ta.Sequence[NodeT]] = self._dict_fac()
-        child_sets_by_node: ta.MutableMapping[ta.Optional[NodeT], ta.AbstractSet[NodeT]] = self._dict_fac()
+        child_sets_by_node: ta.MutableMapping[NodeT | None, ta.AbstractSet[NodeT]] = self._dict_fac()
         parents_by_node: ta.MutableMapping[NodeT, NodeT | None] = self._dict_fac()
 
         children_by_node[None] = [root]
@@ -190,7 +190,7 @@ class BasicTreeAnalysis(ta.Generic[NodeT]):
         e: ta.Any
         d: ta.Any
         if identity:
-            e, d = id, col.unique_dict((id(n), n) for n, _ in pairs)
+            e, d = id, col.unique_map((id(n), n) for n, _ in pairs)
         else:
             e, d = lang.identity, lang.identity
         tsd = {e(n): {e(p)} for n, p in parents_by_node.items()}

omlish/http/__init__.py

@@ -0,0 +1,25 @@
+from . import consts  # noqa
+
+from .cookies import (  # noqa
+    CookieTooBigError,
+    dump_cookie,
+    parse_cookie,
+)
+
+from .dates import (  # noqa
+    http_date,
+    parse_date,
+)
+
+from .encodings import (  # noqa
+    latin1_decode,
+    latin1_encode,
+)
+
+from .json import (  # noqa
+    JSON_TAGGER,
+    JsonTag,
+    JsonTagger,
+    json_dumps,
+    json_loads,
+)

omlish/http/asgi.py

@@ -0,0 +1,132 @@
+import abc
+import logging
+import typing as ta
+import urllib.parse
+
+from .. import check
+from . import consts
+
+
+log = logging.getLogger(__name__)
+
+
+##
+
+
+AsgiScope: ta.TypeAlias = ta.Mapping[str, ta.Any]
+AsgiMessage: ta.TypeAlias = ta.Mapping[str, ta.Any]
+AsgiRecv: ta.TypeAlias = ta.Callable[[], ta.Awaitable[AsgiMessage]]
+AsgiSend: ta.TypeAlias = ta.Callable[[AsgiMessage], ta.Awaitable[None]]
+AsgiApp: ta.TypeAlias = ta.Callable[[AsgiScope, AsgiRecv, AsgiSend], ta.Awaitable[None]]
+AsgiWrapper: ta.TypeAlias = ta.Callable[[AsgiApp, AsgiScope, AsgiRecv, AsgiSend], ta.Awaitable[None]]
+
+
+class AsgiApp_(abc.ABC):  # noqa
+    @abc.abstractmethod
+    async def __call__(self, scope: AsgiScope, recv: AsgiRecv, send: AsgiSend) -> None:
+        raise NotImplementedError
+
+
+##
+
+
+async def stub_lifespan(scope: AsgiScope, recv: AsgiRecv, send: AsgiSend, *, verbose: bool = False) -> None:
+    while True:
+        message = await recv()
+        if message['type'] == 'lifespan.startup':
+            if verbose:
+                log.info('Lifespan starting up')
+            await send({'type': 'lifespan.startup.complete'})
+
+        elif message['type'] == 'lifespan.shutdown':
+            if verbose:
+                log.info('Lifespan shutting down')
+            await send({'type': 'lifespan.shutdown.complete'})
+            return
+
+
+##
+
+
+async def start_response(
+        send: AsgiSend,
+        status: int,
+        content_type: bytes = consts.CONTENT_TYPE_TEXT_UTF8,
+        headers: ta.Sequence[tuple[bytes, bytes]] | None = None,
+) -> None:
+    await send({
+        'type': 'http.response.start',
+        'status': status,
+        'headers': [
+            (b'content-type', content_type),
+            *(headers or ()),
+        ],
+    })
+
+
+async def finish_response(
+        send: AsgiSend,
+        body: bytes = b'',
+) -> None:
+    await send({
+        'type': 'http.response.body',
+        'body': body,
+    })
+
+
+async def send_response(
+        send: AsgiSend,
+        status: int,
+        content_type: bytes = consts.CONTENT_TYPE_TEXT_UTF8,
+        headers: ta.Sequence[tuple[bytes, bytes]] | None = None,
+        body: bytes = b'',
+) -> None:
+    await start_response(
+        send,
+        status=status,
+        content_type=content_type, headers=headers,
+    )
+    await finish_response(
+        send,
+        body=body,
+    )
+
+
+async def redirect_response(
+        send: AsgiSend,
+        url: str,
+        headers: ta.Sequence[tuple[bytes, bytes]] | None = None,
+) -> None:
+    log.info('Redirecting to %s', url)
+    await send({
+        'type': 'http.response.start',
+        'status': 302,
+        'headers': [
+            (b'content-type', consts.CONTENT_TYPE_TEXT_UTF8),
+            (b'location', url.encode()),
+            *(headers or ()),
+        ],
+    })
+    await send({
+        'type': 'http.response.body',
+        'body': b'',
+    })
+
+
+##
+
+
+async def read_body(recv: AsgiRecv) -> bytes:
+    body = b''
+    more_body = True
+    while more_body:
+        message = await recv()
+        body += message.get('body', b'')
+        more_body = message.get('more_body', False)
+    return body
+
+
+async def read_form_body(recv: AsgiRecv) -> dict[bytes, bytes]:
+    body = await read_body(recv)
+    dct = urllib.parse.parse_qs(body)  # noqa
+    return {k: check.single(v) for k, v in dct.items()}