moat-kv 0.70.23__py3-none-any.whl → 0.70.24__py3-none-any.whl

build/lib/docs/source/conf.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Documentation build configuration file, created by
+# sphinx-quickstart on Sat Jan 21 19:11:14 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+# So autodoc can import our package
+sys.path.insert(0, os.path.abspath("../.."))
+
+# Warn about all references to unknown targets
+nitpicky = True
+# Except for these ones, which we expect to point to unknown targets:
+nitpick_ignore = [
+    # Format is ("sphinx reference type", "string"), e.g.:
+    ("py:obj", "bytes-like"),
+    ("py:class", "RangeSet"),
+    ("py:class", "Any"),
+]
+
+autodoc_inherit_docstrings = False
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.coverage",
+    "sphinx.ext.napoleon",
+    "sphinxcontrib_trio",
+]
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+    "trio": ("https://trio.readthedocs.io/en/stable", None),
+}
+
+autodoc_member_order = "bysource"
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = []
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = ".rst"
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = "MoaT-KV"
+copyright = "The MoaT-KV authors"
+author = "The MoaT-KV authors"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+import pkg_resources
+
+version = pkg_resources.get_distribution("moat.kv")._version
+
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# The default language for :: blocks
+highlight_language = "python3"
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# html_theme = 'alabaster'
+
+# We have to set this ourselves, not only because it's useful for local
+# testing, but also because if we don't then RTD will throw away our
+# html_theme_options.
+import sphinx_rtd_theme
+
+html_theme = "sphinx_rtd_theme"
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+html_theme_options = {
+    # default is 2
+    # show deeper nesting in the RTD theme's sidebar TOC
+    # https://stackoverflow.com/questions/27669376/
+    # I'm not 100% sure this actually does anything with our current
+    # versions/settings...
+    "navigation_depth": 4,
+    "logo_only": True,
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "moat.kv-doc"
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, "moat.kv.tex", "Trio Documentation", author, "manual"),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [(master_doc, "moat.kv", "MoaT-KV Documentation", [author], 1)]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (
+        master_doc,
+        "moat.kv",
+        "MoaT-KV Documentation",
+        author,
+        "MoaT-KV",
+        "A distributed no-master key-value store",
+        "Miscellaneous",
+    ),
+]

build/lib/examples/pathify.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+
+# Batch-convert data. In this case I had some entries which were stored as
+# a list, but using Path made much more sense (esp when you need to
+# view/edit the yaml export).
+
+import anyio
+from moat.kv.client import open_client
+from moat.util import P, yload, Path
+import asyncclick as click
+
+
+def conv(m, s: str) -> bool:
+    try:
+        d = m.value[s]
+    except KeyError:
+        return 0
+    if isinstance(d, Path):
+        return 0
+    if not isinstance(d, Sequence):
+        return 0
+    d = Path.build(d)
+    m.value[s] = d
+    return 1
+
+
+@click.command()
+@click.argument("path", type=P)
+@click.argument("keys", type=str, nargs=-1)
+async def main(path, keys):
+    if not keys:
+        keys = "src dest dst state".split()
+    with open("/etc/moat.kv.cfg") as cff:
+        cfg = yload(cff)
+    async with open_client(**cfg) as client:
+        async for m in client.get_tree(path, nchain=2):
+            n = 0
+            for k in keys:
+                n += conv(m, k)
+            if n:
+                await client.set(ORIG + m.path, value=m.value, chain=m.chain)
+
+
+if __name__ == "__main__":
+    main()

build/lib/moat/kv/__init__.py

@@ -0,0 +1,19 @@
+# pylint: disable=W0703,C0103
+from __future__ import annotations
+
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)
+
+try:
+    import warning
+    import pkg_resources  # part of setuptools
+
+    with warnings.filterwarnings("ignore"):
+        _version = pkg_resources.require("moat.kv")[0].version
+    del pkg_resources
+    del warnings
+
+    _version_tuple = tuple(int(x) for x in _version.split("."))
+
+except Exception:  # pragma: no cover
+    _version = "0.0.1"
+    _version_tuple = (0, 0, 1)

build/lib/moat/kv/_cfg.yaml

@@ -0,0 +1,97 @@
+conn:
+  # client: controls how to talk to the MoaT-KV server
+  host: localhost
+  port: 27586
+  ssl: false
+  # ssl:
+  #   cert: '/path/to/cert.pem',key='/path/to/cert.key'
+  init_timeout: 5
+  # time to wait for connection plus greeting
+  auth: null
+  # no auth used by default
+  name: null
+  # defaults to a seqnum
+config:
+  prefix: !P :.moat.kv.config
+errors:
+  prefix: !P :.moat.kv.error
+codes:
+  prefix: !P :.moat.kv.code.proc
+modules:
+  prefix: !P :.moat.kv.code.module
+runner: # for moat.kv.runner.RunnerRoot
+  # storage for runnable commands
+  prefix: !P :.moat.kv.run"
+  # storage for runner states
+  state: !P :.moat.kv.state"
+
+  name: "run"
+  # Serf event name, suffixed by subpath
+
+  start_delay: 1
+  # time to wait between job starts. Not optional.
+
+  ping: -15
+  # set an I-am-running message every those-many seconds
+  # positive: set in moat.kv, negative: broadcast to :moat.kv.run tag
+
+  actor:
+    # Actor config, required for Runner
+    cycle: 20
+    nodes: -1
+    splits: 5
+    n_hosts: 3
+    version: 1
+  sub:
+    # tags for various runner modes
+    group: "any"
+    single: "at"
+    all: "all"
+server:
+  # server-side configuration
+  buffer: 10
+  # per-stream buffer
+
+  backend: "mqtt"
+  # default
+  mqtt:
+    uri: "mqtt://localhost:1883"
+  serf:
+    host: "localhost"
+    port: 7373
+
+  # event message path/topic prefix
+  root: !P moat.kv
+
+  paranoia: False
+  # typecheck server-to-server updates?
+  #
+  # which addresses/ports to accept MoaT-KV connections on
+  bind: [{}]
+  bind_default:
+    # default values for all elements of "bind"
+    host: "localhost"
+    port: PORT
+    ssl: False
+  change:
+    length: 5
+    # chain length: use max nr of network sections +1
+  ping:
+    cycle: 10
+    gap: 2
+    # asyncserf.Actor config timing for server sync
+  # ping also controls minimum server startup time
+  delete:
+    # asyncserf.Actor config timing for deletion
+    cycle: 100
+    gap: 10
+    version: 1
+paranoia: false
+# typecheck server>server updates?
+
+# how does a new server reach existing nodes, to download state?
+domain: null
+# domain in which to look up node names, if not in hostmap
+hostmap:   # map MoaT-KV server names to connect destinations
+  test1: ["localhost", 27586]
+  test2: ["does-not-exist.invalid", 27586]

build/lib/moat/kv/_main.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Basic DistKV support
+
+"""
+
+from __future__ import annotations
+
+import logging
+
+import asyncclick as click
+from moat.util import attrdict, combine_dict, load_subgroup, CFG, ensure_cfg
+
+from moat.kv.auth import gen_auth
+from moat.kv.client import client_scope
+
+logger = logging.getLogger(__name__)
+
+
+ensure_cfg("moat.kv")
+
+
+class NullObj:
+    """
+    This helper defers raising an exception until one of its attributes is
+    actually accessed.
+    """
+
+    def __init__(self, exc):
+        self._exc = exc
+
+    def __call__(self, *a, **kw):
+        raise self._exc
+
+    def __await__(self):
+        raise self._exc
+
+    def __getattr__(self, k):
+        if k[0] == "_" and k not in ("_request", "_cfg"):
+            return object.__getattribute__(self, k)
+        raise self._exc
+
+
+@load_subgroup(sub_pre="moat.kv.command", sub_post="cli", ext_pre="moat.kv", ext_post="_main.cli")
+@click.option("-h", "--host", default=None, help=f"Host to use. Default: {CFG.kv.conn.host}")
+@click.option(
+    "-p",
+    "--port",
+    type=int,
+    default=None,
+    help=f"Port to use. Default: {CFG.kv.conn.port}",
+)
+@click.option(
+    "-a",
+    "--auth",
+    type=str,
+    default=None,
+    help="Auth params. =file or 'type param=value…' Default: _anon",
+)
+@click.option("-m", "--metadata", is_flag=True, help="Include/print metadata.")
+@click.pass_context
+async def cli(ctx, host, port, auth, metadata):
+    """The MoaT Key-Value subsystem.
+
+    All commands (except 'server' and 'dump') connect to a MoaT-KV server.
+    """
+    obj = ctx.obj
+    cfg = attrdict()
+    if host is not None:
+        cfg.host = host
+    if port is not None:
+        cfg.port = port
+
+    if auth is not None:
+        cfg.auth = gen_auth(auth)
+        if obj.DEBUG:
+            cfg.auth._DEBUG = True
+
+    cfg = combine_dict(attrdict(kv=attrdict(conn=cfg)), obj.cfg, cls=attrdict)
+
+    obj.meta = 3 if metadata else False
+
+    try:
+        if ctx.invoked_subcommand in {None, "server", "dump"}:
+            obj.client = NullObj(RuntimeError("Not a client command"))
+        else:
+            obj.client = await client_scope(**cfg.kv)
+    except OSError as exc:
+        obj.client = NullObj(exc)
+    else:
+        logger.debug("Connected.")

build/lib/moat/kv/actor/__init__.py

@@ -0,0 +1,98 @@
+"""
+This module implements a :class:`asyncactor.Actor` which works on top of
+a MoaT-KV client.
+"""
+from __future__ import annotations
+
+from asyncactor import Actor
+from asyncactor.abc import MonitorStream, Transport
+
+__all__ = [
+    "ClientActor",
+    "ActorState",
+    "BrokenState",
+    "DetachedState",
+    "PartialState",
+    "CompleteState",
+]
+
+
+class ClientActor(Actor):
+    def __init__(self, client, *a, topic, **kw):
+        super().__init__(ClientTransport(client, topic), *a, **kw)
+
+
+class ClientTransport(Transport):
+    """
+    This class exports the client's direct messaging interface to the
+    actor.
+    """
+
+    def __init__(self, client, topic):
+        self.client = client
+        self.topic = topic
+
+    def monitor(self):
+        return ClientMonitor(self)
+
+    async def send(self, payload):
+        await self.client.msg_send(self.topic, payload)
+
+
+class ClientMonitor(MonitorStream):
+    _mon1 = None
+    _mon2 = None
+    _it = None
+
+    async def __aenter__(self):
+        self._mon1 = self.transport.client.msg_monitor(self.transport.topic)
+        self._mon2 = await self._mon1.__aenter__()
+        return self
+
+    async def __aexit__(self, *tb):
+        return await self._mon1.__aexit__(*tb)
+
+    def __aiter__(self):
+        self._it = self._mon2.__aiter__()
+        return self
+
+    async def __anext__(self):
+        msg = await self._it.__anext__()
+        return msg.data
+
+
+# The following events are used by Runner etc. to notify running jobs
+# about the current connectivity state.
+#
+class ActorState:
+    """base class for states"""
+
+    def __init__(self, msg=None):
+        self.msg = msg
+
+    def __repr__(self):
+        return "<%s:%r>" % (self.__class__.__name__, self.msg)
+
+
+class BrokenState(ActorState):
+    """I have no idea what's happening, probably nothing good"""
+
+    pass
+
+
+class DetachedState(ActorState):
+    """I am detached, my actor group is not visible"""
+
+    pass
+
+
+class PartialState(ActorState):
+    """Some but not all members of my actor group are visible"""
+
+    pass
+
+
+class CompleteState(ActorState):
+    """All members of my actor group are visible"""
+
+    pass

build/lib/moat/kv/actor/deletor.py

@@ -0,0 +1,139 @@
+"""
+This module implements additional code for the server-side DeleteActor,
+which is used to clean up the list of deleted nodes.
+"""
+
+from __future__ import annotations
+
+import weakref
+from collections import deque
+
+import anyio
+from asyncactor import Actor, PingEvent, TagEvent
+from asyncactor.backend import get_transport
+
+TAGS = 4
+
+
+class DeleteActor:
+    _enabled = None
+
+    def __init__(self, server):
+        self._server = weakref.ref(server)
+        self.deleted = deque()
+        self.tags = []
+        self.actor = None
+
+        self.max_seen = 0
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = 0
+
+    @property
+    def server(self):
+        return self._server()
+
+    async def tock_me(self):
+        """
+        Add the current tock to our buffer.
+
+        This is updated whenever a new leader is selected.
+        """
+        self.tags.append(self.server.tock)
+        self.tags = self.tags[-TAGS:]
+        await self.actor.set_value((self.tags[0], self.tags[-1]))
+
+    def add_deleted(self, nodes: NodeSet):  # noqa: F821
+        """
+        These nodes are deleted. Remember them for some time.
+        """
+        if self.n_nodes == 0:
+            return
+        self.deleted.append((self.server.tock, nodes))
+
+    def purge_to(self, tock):
+        """
+        Sufficient time has passed since this tock was seen, while all
+        Delete actor nodes were active. Finally flush the entries that have
+        been deleted before it.
+        """
+        while self.deleted and self.deleted[0][0] < tock:
+            d = self.deleted.popleft()
+            self.server.purge_deleted(d[1])
+
+    async def enable(self, n):
+        """
+        Enable this actor, as a group of N.
+        """
+        if self.actor is None:
+            self._enabled = True
+        else:
+            await self.actor.enable(n)
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = n
+
+    async def disable(self, n: int = 0):
+        """
+        Disable this actor. It will still listen, and require N Delete
+        actor members in order to flush its deletion entries.
+
+        Completely disable deletion flushing by passing n=0.
+        """
+        if self.actor is None:
+            self._enabled = False
+        else:
+            await self.actor.disable()
+        self.n_tags = 0
+        self.n_pings = 0
+        self.n_nodes = n
+
+    async def run(self, evt: anyio.abc.Event = None):
+        """
+        The task that monitors the Delete actor.
+        """
+        try:
+            T = get_transport("moat_kv")
+            async with Actor(
+                T(self.server.backend, *self.server.cfg.server.root, "del"),
+                name=self.server.node.name,
+                cfg=self.server.cfg.server.delete,
+                enabled=False,
+            ) as actor:
+                self.actor = actor
+                if self._enabled is not None:
+                    if self._enabled:
+                        await actor.enable()
+                    else:
+                        await actor.disable()
+                if evt is not None:
+                    evt.set()
+                async for evt in actor:
+                    if isinstance(evt, PingEvent):
+                        val = evt.value
+                        if val is None:
+                            self.n_pings = self.n_tags = 0
+                            continue
+                        if len(evt.msg.history) < self.n_nodes:
+                            self.n_pings = self.n_tags = 0
+                            continue
+                        self.n_pings += 1
+                        if self.n_pings > self.n_nodes:
+                            mx, self.max_seen = (
+                                self.max_seen,
+                                max(self.max_seen, val[1]),
+                            )
+                            if val[0] > mx > 0:
+                                await self.server.resync_deleted(evt.msg.history)
+                                continue
+                            self.purge_to(val[0])
+                            self.max_seen = max(self.max_seen, val[1])
+
+                    elif isinstance(evt, TagEvent):
+                        if actor.history_size == self.n_nodes:
+                            self.n_tags += 1
+                            if self.n_tags > 2:
+                                self.purge_to(self.tags[0])
+                            await self.tock_me()
+        finally:
+            self.actor = None