nomosdb 0.16.0__py3-none-any.whl

nomosdb/__init__.py

@@ -0,0 +1,59 @@
+"""
+NomosDB Python driver.
+
+Bolt (recommended): real transactions, retries, cluster routing, asyncio.
+
+    >>> import nomosdb
+    >>> drv = nomosdb.driver("bolt://localhost:7687", auth=("nomosdb", "secret"))
+    >>> with drv.session() as s:
+    ...     for rec in s.run("MATCH (p:Person) RETURN p.name AS name LIMIT 10"):
+    ...         print(rec["name"])
+    >>> drv.close()
+
+The earlier client stays available: NomosDB("bolt://...") or NomosDB("http://...")
+with query() returning dict rows, and the ORM.
+"""
+
+from ._async.driver import AsyncDriver, async_driver
+from ._async.work import AsyncManagedTransaction, AsyncResult, AsyncSession, AsyncTransaction
+from ._common import READ_ACCESS, WRITE_ACCESS, Record, ResultSummary, SummaryCounters
+from ._sync.driver import Driver, driver
+from ._sync.work import ManagedTransaction, Result, Session, Transaction
+from ._version import __version__
+from .client import NomosDB
+from .connection import Result as QueryResult
+from .exceptions import (
+    AuthError,
+    AuthenticationError,
+    ClientError,
+    ConfigurationError,
+    ConnectionPoolFull,
+    DatabaseError,
+    NomosConnectionError,
+    NomosDBError,
+    NotALeader,
+    ProtocolError,
+    QueryException,
+    ResultConsumedError,
+    ResultNotSingleError,
+    ServerError,
+    ServiceUnavailable,
+    SessionExpired,
+    TransactionCommitUnknown,
+    TransactionError,
+    TransientError,
+)
+from .orm import Entity, Node, Relationship, node_class, relationship_class
+
+__all__ = [
+    "driver", "async_driver", "Driver", "AsyncDriver",
+    "Session", "AsyncSession", "Transaction", "AsyncTransaction",
+    "ManagedTransaction", "AsyncManagedTransaction", "Result", "AsyncResult",
+    "Record", "ResultSummary", "SummaryCounters", "READ_ACCESS", "WRITE_ACCESS",
+    "NomosDB", "QueryResult", "Node", "Relationship", "Entity", "node_class", "relationship_class",
+    "NomosDBError", "NomosConnectionError", "ConnectionPoolFull", "QueryException", "TransactionError",
+    "AuthenticationError", "ProtocolError", "ConfigurationError", "ServerError", "ClientError",
+    "TransientError", "DatabaseError", "AuthError", "NotALeader", "ServiceUnavailable", "SessionExpired",
+    "TransactionCommitUnknown", "ResultConsumedError", "ResultNotSingleError",
+    "__version__",
+]

nomosdb/_async/connection.py

@@ -0,0 +1,126 @@
+"""One Bolt connection: handshake, authentication, pipelined requests."""
+
+from typing import Optional, Tuple
+
+from .._bolt import HANDSHAKE, Dechunker, Protocol, Response, parse_version
+from .._common import Address, Config
+from .._io import AsyncBoltSocket, monotonic
+from ..exceptions import ServerError, ServiceUnavailable, SessionExpired
+
+
+class AsyncConnection:
+    def __init__(self, sock: AsyncBoltSocket, address: Address, version: Tuple[int, int]):
+        self._sock = sock
+        self.address = address
+        self.protocol = Protocol(version)
+        self._dechunker = Dechunker()
+        self._inbox: list = []
+        self.server_agent = ""
+        self.connection_id = ""
+        self.defunct = False
+        self.closed = False
+        self.last_used = monotonic()
+
+    @classmethod
+    async def open(cls, address: Address, config: Config) -> "AsyncConnection":
+        sock = await AsyncBoltSocket.connect(address, config.ssl_context, config.connection_timeout)
+        try:
+            await sock.sendall(HANDSHAKE)
+            answer = b""
+            while len(answer) < 4:
+                part = await sock.recv(config.connection_timeout)
+                if not part:
+                    raise ServiceUnavailable(f"{address[0]}:{address[1]} closed the connection during the Bolt handshake "
+                                             "(is it the Bolt port? nomosdb-server listens for Bolt on 7687)")
+                answer += part
+            conn = cls(sock, address, parse_version(answer[:4]))
+            if len(answer) > 4:
+                conn._inbox.extend(conn._dechunker.feed(answer[4:]))
+            hello = conn.protocol.hello(config.user_agent, config.auth, config.routing_context)
+            await conn.send_all()
+            await conn.fetch_all()
+        except BaseException as e:
+            await sock.close()
+            if isinstance(e, OSError):
+                raise ServiceUnavailable(f"{address[0]}:{address[1]} did not complete the Bolt handshake "
+                                         f"(is it the Bolt port? nomosdb-server listens for Bolt on 7687): {e}") from None
+            raise
+        conn.server_agent = str(hello.metadata.get("server", ""))
+        conn.connection_id = str(hello.metadata.get("connection_id", ""))
+        return conn
+
+    # -- I/O
+    async def send_all(self) -> None:
+        data = self.protocol.take_outbox()
+        if not data:
+            return
+        try:
+            await self._sock.sendall(data)
+        except OSError as e:
+            await self._broken(e)
+
+    async def fetch_message(self) -> None:
+        """Reads until at least one answer has been handled."""
+        while not self._inbox:
+            try:
+                data = await self._sock.recv()
+            except OSError as e:
+                await self._broken(e)
+            if not data:
+                await self._broken(None)
+            self._inbox.extend(self._dechunker.feed(data))
+        tag, fields = self._inbox.pop(0)
+        self.protocol.handle(tag, fields)
+
+    async def fetch_all(self) -> None:
+        """Reads the answers to every request sent; raises the first failure."""
+        error: Optional[ServerError] = None
+        while self.protocol.pending:
+            r = self.protocol.pending[0]
+            while not r.done:
+                await self.fetch_message()
+            if r.error is not None and error is None:
+                error = r.error
+        if error is not None:
+            raise error
+
+    async def wait(self, r: Response) -> Response:
+        """Sends what is queued and reads until r is answered; raises its failure."""
+        await self.send_all()
+        while not r.done:
+            await self.fetch_message()
+        if r.error is not None:
+            raise r.error
+        return r
+
+    async def _broken(self, e: Optional[BaseException]) -> None:
+        self.defunct = True
+        await self._sock.close()
+        why = f": {e}" if e else ""
+        raise SessionExpired(f"the connection to {self.address[0]}:{self.address[1]} broke{why}")
+
+    # -- state
+    async def reset(self) -> None:
+        """Back to READY: ends a transaction or a failure."""
+        r = self.protocol.reset(Response("RESET"))
+        await self.wait(r)
+        await self.fetch_all()
+
+    @property
+    def ready(self) -> bool:
+        return not (self.defunct or self.closed or self.protocol.pending or self.protocol.failed)
+
+    async def close(self) -> None:
+        if self.closed:
+            return
+        self.closed = True
+        if not self.defunct:
+            try:
+                self.protocol.goodbye()
+                await self._sock.sendall(self.protocol.take_outbox())
+            except OSError:
+                pass
+        await self._sock.close()
+
+    def __repr__(self) -> str:
+        return f"<AsyncConnection {self.address[0]}:{self.address[1]} bolt/{self.protocol.version[0]}.{self.protocol.version[1]}>"

nomosdb/_async/driver.py

@@ -0,0 +1,75 @@
+"""The driver: configuration, the connection pool, sessions."""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from .._common import READ_ACCESS, WRITE_ACCESS, Config, Record, ResultSummary
+from ..exceptions import ConfigurationError
+from .pool import AsyncDirectPool, AsyncRoutingPool
+from .work import AsyncManagedTransaction, AsyncSession
+
+
+class AsyncDriver:
+    """Holds the connection pool of one server or cluster. Thread safe (the blocking
+    Driver) / task safe (AsyncDriver); sessions are not."""
+
+    def __init__(self, uri: str, auth: Any = None, **config: Any):
+        self._config = Config(uri, auth, **config)
+        self._pool = AsyncRoutingPool(self._config) if self._config.routing else AsyncDirectPool(self._config)
+
+    @property
+    def encrypted(self) -> bool:
+        return self._config.ssl_context is not None
+
+    def session(self, default_access_mode: str = WRITE_ACCESS, bookmarks: Optional[List[str]] = None,
+                fetch_size: Optional[int] = None, database: Optional[str] = None) -> AsyncSession:
+        return AsyncSession(self._pool, self._config, default_access_mode, bookmarks, fetch_size, database)
+
+    async def verify_connectivity(self) -> None:
+        """Raises unless a server answers and accepts the credentials."""
+        conn = await self._pool.acquire(READ_ACCESS, [])
+        await self._pool.release(conn)
+
+    async def get_server_info(self) -> Dict[str, Any]:
+        conn = await self._pool.acquire(READ_ACCESS, [])
+        try:
+            return {"address": conn.address, "agent": conn.server_agent,
+                    "protocol_version": conn.protocol.version, "connection_id": conn.connection_id}
+        finally:
+            await self._pool.release(conn)
+
+    async def execute_query(self, query: str, parameters: Optional[Dict[str, Any]] = None, routing: str = "w",
+                            bookmarks: Optional[List[str]] = None,
+                            **kwparameters: Any) -> Tuple[List[Record], ResultSummary, List[str]]:
+        """Runs one statement in a retried transaction and returns (records, summary, keys)."""
+        if routing not in ("r", "w"):
+            raise ConfigurationError("routing is 'r' (a reader) or 'w' (the leader)")
+        params = {**(parameters or {}), **kwparameters}
+
+        async def work(tx: AsyncManagedTransaction) -> Tuple[List[Record], ResultSummary, List[str]]:
+            res = await tx.run(query, params)
+            records = await res.to_list()
+            return records, await res.consume(), await res.keys()
+
+        async with self.session(bookmarks=bookmarks) as s:
+            if routing == "r":
+                return await s.execute_read(work)
+            return await s.execute_write(work)
+
+    async def close(self) -> None:
+        await self._pool.close()
+
+    async def __aenter__(self) -> "AsyncDriver":
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        await self.close()
+
+
+def async_driver(uri: str, auth: Any = None, **config: Any) -> AsyncDriver:
+    """A driver for bolt://host:7687 (one server) or nomosdb://host:7687 (a cluster),
+    with +s (TLS, verified) or +ssc (TLS, self-signed) for encryption.
+
+    auth is (user, password). Options: max_connection_pool_size, connection_timeout,
+    connection_acquisition_timeout, max_transaction_retry_time, fetch_size,
+    trusted_certificates (a CA file for +s), user_agent."""
+    return AsyncDriver(uri, auth, **config)

nomosdb/_async/pool.py

@@ -0,0 +1,228 @@
+"""Connection pools: one server (bolt://) or a cluster through its routing table (nomosdb://)."""
+
+from collections import deque
+from typing import Deque, Dict, List, Optional
+
+from .._bolt import Response
+from .._common import READ_ACCESS, Address, Config, parse_address
+from .._io import AsyncCondition, monotonic
+from ..exceptions import ClientError, ServiceUnavailable
+from .connection import AsyncConnection
+
+
+class AsyncPool:
+    """Connections per address, at most max_connection_pool_size each."""
+
+    def __init__(self, config: Config):
+        self.config = config
+        self._idle: Dict[Address, Deque[AsyncConnection]] = {}
+        self._in_use: Dict[Address, int] = {}
+        self._cond = AsyncCondition()
+        self.closed = False
+
+    async def acquire_at(self, address: Address) -> AsyncConnection:
+        if self.closed:
+            raise ServiceUnavailable("the driver is closed")
+        deadline = monotonic() + self.config.connection_acquisition_timeout
+        while True:
+            async with self._cond:
+                idle = self._idle.setdefault(address, deque())
+                conn = idle.pop() if idle else None
+                if conn is None:
+                    if self._in_use.get(address, 0) < self.config.max_connection_pool_size:
+                        self._in_use[address] = self._in_use.get(address, 0) + 1
+                        opening = True
+                    else:
+                        opening = False
+                        left = deadline - monotonic()
+                        if left <= 0 or not await self._cond.wait_for(
+                                lambda: self._idle.get(address) or
+                                self._in_use.get(address, 0) < self.config.max_connection_pool_size, left):
+                            raise ServiceUnavailable(
+                                f"no connection to {address[0]}:{address[1]} became free within "
+                                f"{self.config.connection_acquisition_timeout} s (max_connection_pool_size="
+                                f"{self.config.max_connection_pool_size})")
+                        continue
+                else:
+                    self._in_use[address] = self._in_use.get(address, 0) + 1
+                    opening = False
+            if opening:
+                try:
+                    return await AsyncConnection.open(address, self.config)
+                except BaseException:
+                    await self._forget(address)
+                    raise
+            if await self._alive(conn):
+                return conn
+            await self._forget(address)
+
+    async def _alive(self, conn: AsyncConnection) -> bool:
+        t = self.config.liveness_check_timeout
+        if conn.defunct or conn.closed:
+            return False
+        if t is not None and monotonic() - conn.last_used > t:
+            try:
+                await conn.reset()
+            except (ServiceUnavailable, ClientError):
+                await conn.close()
+                return False
+        return True
+
+    async def _forget(self, address: Address) -> None:
+        async with self._cond:
+            self._in_use[address] -= 1
+            self._cond.notify_all()
+
+    async def release(self, conn: AsyncConnection) -> None:
+        """Takes a connection back; one in a transaction or a failure is reset first."""
+        if not (conn.defunct or conn.closed) and not conn.ready:
+            try:
+                await conn.reset()
+            except Exception:
+                await conn.close()
+        async with self._cond:
+            self._in_use[conn.address] -= 1
+            if conn.defunct or conn.closed or self.closed:
+                keep = False
+            else:
+                conn.last_used = monotonic()
+                self._idle.setdefault(conn.address, deque()).append(conn)
+                keep = True
+            self._cond.notify_all()
+        if not keep:
+            await conn.close()
+
+    async def deactivate(self, address: Address) -> None:
+        """Closes the idle connections to a server that failed."""
+        async with self._cond:
+            idle = self._idle.pop(address, deque())
+        for c in idle:
+            await c.close()
+
+    async def close(self) -> None:
+        self.closed = True
+        async with self._cond:
+            idle = [c for q in self._idle.values() for c in q]
+            self._idle.clear()
+        for c in idle:
+            await c.close()
+
+
+class AsyncDirectPool(AsyncPool):
+    """bolt://: every session goes to the one server."""
+
+    async def acquire(self, access_mode: str, bookmarks: List[str]) -> AsyncConnection:
+        return await self.acquire_at(self.config.address)
+
+    async def on_write_failure(self, address: Address) -> None:
+        pass
+
+    async def on_unavailable(self, address: Address) -> None:
+        await self.deactivate(address)
+
+
+class _RoutingTable:
+    def __init__(self, seed: Address):
+        self.routers: List[Address] = [seed]
+        self.readers: List[Address] = []
+        self.writers: List[Address] = []
+        self.expires = 0.0
+        self._next_reader = 0
+        self._next_writer = 0
+
+    def fresh(self, access_mode: str) -> bool:
+        servers = self.readers if access_mode == READ_ACCESS else self.writers
+        return monotonic() < self.expires and bool(servers)
+
+    def update(self, rt: Dict) -> None:
+        ttl = float(rt.get("ttl", 300))
+        readers, writers, routers = [], [], []
+        for s in rt.get("servers", []):
+            addrs = [parse_address(a) for a in s.get("addresses", [])]
+            role = s.get("role")
+            if role == "READ":
+                readers += addrs
+            elif role == "WRITE":
+                writers += addrs
+            elif role == "ROUTE":
+                routers += addrs
+        if not routers:
+            raise ServiceUnavailable("the routing table names no routers")
+        self.routers, self.readers, self.writers = routers, readers, writers
+        self.expires = monotonic() + ttl
+
+    def pick(self, access_mode: str) -> Optional[Address]:
+        if access_mode == READ_ACCESS:
+            if not self.readers:
+                return None
+            self._next_reader = (self._next_reader + 1) % len(self.readers)
+            return self.readers[self._next_reader]
+        if not self.writers:
+            return None
+        self._next_writer = (self._next_writer + 1) % len(self.writers)
+        return self.writers[self._next_writer]
+
+    def remove(self, address: Address) -> None:
+        for lst in (self.routers, self.readers, self.writers):
+            while address in lst:
+                lst.remove(address)
+
+    def remove_writer(self, address: Address) -> None:
+        while address in self.writers:
+            self.writers.remove(address)
+
+
+class AsyncRoutingPool(AsyncPool):
+    """nomosdb:// / neo4j://: asks the cluster which members write and read (ROUTE)."""
+
+    def __init__(self, config: Config):
+        super().__init__(config)
+        self.table = _RoutingTable(config.address)
+        self._refreshing = AsyncCondition()
+
+    async def acquire(self, access_mode: str, bookmarks: List[str]) -> AsyncConnection:
+        last: Optional[BaseException] = None
+        for _ in range(3):
+            if not self.table.fresh(access_mode):
+                await self._refresh(bookmarks)
+            address = self.table.pick(access_mode)
+            if address is None:
+                raise ServiceUnavailable(
+                    f"the cluster has no {'reader' if access_mode == READ_ACCESS else 'leader'} at the moment")
+            try:
+                return await self.acquire_at(address)
+            except ServiceUnavailable as e:
+                last = e
+                await self.on_unavailable(address)
+        raise last if last else ServiceUnavailable("no cluster member could be reached")
+
+    async def _refresh(self, bookmarks: List[str]) -> None:
+        async with self._refreshing:
+            routers = list(self.table.routers)
+            if self.config.address not in routers:
+                routers.append(self.config.address)  # the seed, in case every router moved
+            errors = []
+            for address in routers:
+                try:
+                    conn = await self.acquire_at(address)
+                except ServiceUnavailable as e:
+                    errors.append(str(e))
+                    continue
+                try:
+                    r = conn.protocol.route(self.config.routing_context or {}, list(bookmarks), Response("ROUTE"))
+                    await conn.wait(r)
+                    self.table.update(r.metadata.get("rt", {}))
+                    return
+                except (ServiceUnavailable, ClientError) as e:
+                    errors.append(f"{address[0]}:{address[1]}: {e}")
+                finally:
+                    await self.release(conn)
+            raise ServiceUnavailable("no routing table: " + "; ".join(errors))
+
+    async def on_write_failure(self, address: Address) -> None:
+        """A write reached a member that is not the leader: ask again."""
+        self.table.remove_writer(address)
+
+    async def on_unavailable(self, address: Address) -> None:
+        self.table.remove(address)
+        await self.deactivate(address)