sightingdb-client 0.0.1__py3-none-any.whl

sightingdb/__init__.py

@@ -0,0 +1,85 @@
+"""The Python client for SightingDB.
+
+SightingDB counts things: how many times a value was seen, when it was first
+seen, when it was last seen, and how many namespaces hold it. This library is
+that REST API as Python objects.
+
+    import sightingdb
+
+    with sightingdb.SightingDB("https://localhost:9999", apikey="changeme") as db:
+        db.write("feeds/misp/ips", "127.0.0.1")
+        attribute = db.read("feeds/misp/ips", "127.0.0.1")
+        print(attribute.count, attribute.first_seen_at)
+
+The same API is available for asyncio as
+:class:`~sightingdb.AsyncSightingDB`, with every call awaited.
+"""
+
+from .__version__ import (
+    __author__,
+    __author_email__,
+    __description__,
+    __license__,
+    __url__,
+    __version__,
+)
+from ._async_client import AsyncSightingDB, AsyncWriteBatch
+from ._client import SightingDB, WriteBatch
+from .config import Config
+from .errors import (
+    APIError,
+    AuthenticationError,
+    BadRequestError,
+    BulkWriteError,
+    ConfigurationError,
+    NotFoundError,
+    NotImplementedError,
+    PermissionDeniedError,
+    ProtocolError,
+    ServerError,
+    SightingDBError,
+    TimeoutError,
+    TransportError,
+)
+from .models import (
+    Attribute,
+    BulkWriteResult,
+    Miss,
+    ReadResult,
+    ServerInfo,
+    Sighting,
+    WriteError,
+)
+
+__all__ = [
+    # clients
+    "SightingDB",
+    "AsyncSightingDB",
+    "WriteBatch",
+    "AsyncWriteBatch",
+    "Config",
+    # models
+    "Sighting",
+    "Attribute",
+    "Miss",
+    "ReadResult",
+    "BulkWriteResult",
+    "WriteError",
+    "ServerInfo",
+    # errors
+    "SightingDBError",
+    "ConfigurationError",
+    "TransportError",
+    "TimeoutError",
+    "ProtocolError",
+    "APIError",
+    "BadRequestError",
+    "AuthenticationError",
+    "PermissionDeniedError",
+    "NotFoundError",
+    "NotImplementedError",
+    "ServerError",
+    "BulkWriteError",
+    # metadata
+    "__version__",
+]

sightingdb/__version__.py

@@ -0,0 +1,10 @@
+"""Version and details for sightingdb"""
+
+__description__ = "The Python Client Library for SightingDB"
+__url__ = "https://www.github.com/stricaud/sightingdb-client"
+__version__ = "0.0.1"
+__author__ = "Sebastien Tricaud"
+__author_email__ = "sebastien@honeynet.org"
+__license__ = "MIT"
+__maintainer__ = __author__
+__maintainer_email__ = __author_email__

sightingdb/_async_client.py

@@ -0,0 +1,306 @@
+"""The asynchronous client: the same API as :class:`~sightingdb._client.SightingDB`, awaited."""
+
+from __future__ import annotations
+
+import asyncio
+import itertools
+from types import TracebackType
+from typing import Any, Iterable
+
+import httpx
+
+from . import _protocol as proto
+from ._client import USER_AGENT, _chunks
+from .config import Config
+from .errors import BulkWriteError, NotFoundError, SightingDBError
+from .models import Attribute, BulkWriteResult, ReadResult, ServerInfo, Sighting
+
+__all__ = ["AsyncSightingDB", "AsyncWriteBatch"]
+
+
+class AsyncSightingDB:
+    """A connection to a SightingDB server, for asyncio.
+
+    Every method matches :class:`~sightingdb.SightingDB` — same arguments, same
+    return types — but is awaited, and requests issued together with
+    ``asyncio.gather`` share one connection pool.
+
+        >>> async with AsyncSightingDB(apikey="changeme") as db:
+        ...     await db.write("feeds/ips", "127.0.0.1")
+        ...     found = await db.read_many([("feeds/ips", "127.0.0.1")])
+
+    Args:
+        http_client: An ``httpx.AsyncClient`` to use instead of the one this
+            client would build. Yours is not closed by :meth:`aclose`.
+
+    See :class:`~sightingdb.SightingDB` for the rest.
+    """
+
+    def __init__(
+        self,
+        url: str | None = None,
+        *,
+        host: str | None = None,
+        port: int | None = None,
+        ssl: bool | None = None,
+        apikey: str | None = None,
+        verify: bool | str | None = None,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        config: Config | None = None,
+        config_file: str | None = None,
+        use_env: bool = True,
+        use_file: bool = True,
+        http_client: httpx.AsyncClient | None = None,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ) -> None:
+        self.config = config or Config.resolve(
+            url=url,
+            host=host,
+            port=port,
+            ssl=ssl,
+            apikey=apikey,
+            verify=verify,
+            timeout=timeout,
+            max_retries=max_retries,
+            config_file=config_file,
+            use_env=use_env,
+            use_file=use_file,
+        )
+        self._owns_http = http_client is None
+        self._http = http_client or httpx.AsyncClient(
+            base_url=self.config.url,
+            headers=self._headers(),
+            verify=self.config.verify,
+            timeout=self.config.timeout,
+            transport=transport,
+        )
+
+    def _headers(self) -> dict[str, str]:
+        headers = {"user-agent": USER_AGENT, "accept": "application/json"}
+        if self.config.apikey is not None:
+            headers["authorization"] = self.config.apikey
+        return headers
+
+    # -- lifecycle -------------------------------------------------------
+
+    async def aclose(self) -> None:
+        """Close the connection pool, unless the ``httpx.AsyncClient`` is yours."""
+        if self._owns_http:
+            await self._http.aclose()
+
+    async def __aenter__(self) -> AsyncSightingDB:
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        await self.aclose()
+
+    def __repr__(self) -> str:
+        return f"<AsyncSightingDB {self.config.url}>"
+
+    # -- writing ---------------------------------------------------------
+
+    async def write(
+        self,
+        namespace: str,
+        value: str,
+        *,
+        timestamp: Any = None,
+        ttl: Any = None,
+    ) -> int:
+        """Record one sighting, and return how many times it has been seen."""
+        return await self._send(
+            proto.write(namespace, value, timestamp=timestamp, ttl=ttl)
+        )
+
+    async def write_many(
+        self,
+        sightings: Iterable[Sighting | tuple[str, str] | dict[str, Any]],
+        *,
+        strict: bool = True,
+        chunk_size: int | None = None,
+    ) -> BulkWriteResult:
+        """Record many sightings in one request."""
+        items = proto.coerce_sightings(sightings)
+        if not items:
+            return BulkWriteResult(written=0)
+
+        result = BulkWriteResult(written=0)
+        for chunk in _chunks(items, chunk_size):
+            result = result + await self._send(proto.write_many(chunk))
+        if strict and result.errors:
+            raise BulkWriteError(result)
+        return result
+
+    def batch(
+        self, *, chunk_size: int | None = None, strict: bool = True
+    ) -> AsyncWriteBatch:
+        """Collect sightings and send them together.
+
+            >>> async with db.batch(chunk_size=1000) as batch:
+            ...     async for value in feed:
+            ...         await batch.add("feeds/misp/ips", value)
+        """
+        return AsyncWriteBatch(self, chunk_size=chunk_size, strict=strict)
+
+    # -- reading ---------------------------------------------------------
+
+    async def read(
+        self,
+        namespace: str,
+        value: str,
+        *,
+        stats: bool = False,
+        shadow: bool = True,
+    ) -> Attribute:
+        """Read one value. Raises :class:`~sightingdb.errors.NotFoundError` on a miss."""
+        return await self._send(
+            proto.read(namespace, value, stats=stats, shadow=shadow)
+        )
+
+    async def read_many(
+        self,
+        sightings: Iterable[Sighting | tuple[str, str] | dict[str, Any]],
+        *,
+        stats: bool = False,
+        chunk_size: int | None = None,
+    ) -> list[ReadResult]:
+        """Read many values in one request, a miss being a result rather than an error."""
+        items = proto.coerce_sightings(sightings)
+        if not items:
+            return []
+        results: list[ReadResult] = []
+        for chunk in _chunks(items, chunk_size):
+            results.extend(await self._send(proto.read_many(chunk, stats=stats)))
+        return results
+
+    async def list_values(
+        self, namespace: str, *, shadow: bool = True
+    ) -> list[Attribute]:
+        """Every value in a namespace."""
+        return await self._send(proto.list_values(namespace, shadow=shadow))
+
+    async def exists(self, namespace: str, value: str, *, shadow: bool = True) -> bool:
+        """Whether a value has ever been seen in a namespace."""
+        results = await self.read_many([Sighting(namespace, value, noshadow=not shadow)])
+        return bool(results and results[0].found)
+
+    # -- namespaces ------------------------------------------------------
+
+    async def delete(self, namespace: str, *, missing_ok: bool = False) -> None:
+        """Delete a namespace and everything in it."""
+        try:
+            await self._send(proto.delete(namespace))
+        except NotFoundError:
+            if not missing_ok:
+                raise
+
+    async def info(self) -> ServerInfo:
+        """What the server reports about itself."""
+        return await self._send(proto.info())
+
+    async def ping(self) -> bool:
+        """Whether the server is reachable and answering."""
+        try:
+            await self.info()
+        except SightingDBError:
+            return False
+        return True
+
+    # -- transport -------------------------------------------------------
+
+    async def _send(self, request: proto.Request) -> Any:
+        max_retries = self.config.max_retries
+        for attempt in itertools.count():
+            try:
+                response = await self._http.request(
+                    request.method,
+                    request.path,
+                    params=request.params or None,
+                    json=request.json,
+                )
+            except httpx.HTTPError as exc:
+                if attempt < max_retries and proto.retry_after_exception(
+                    exc, idempotent=request.idempotent
+                ):
+                    await asyncio.sleep(proto.backoff(attempt))
+                    continue
+                raise proto.transport_error(
+                    exc, request, f"{self.config.url}{request.path}"
+                ) from exc
+
+            if attempt < max_retries and proto.retry_after_status(
+                response.status_code, idempotent=request.idempotent
+            ):
+                await asyncio.sleep(proto.backoff(attempt, response))
+                continue
+            return proto.parse(response, request)
+        raise AssertionError("unreachable")  # pragma: no cover
+
+
+class AsyncWriteBatch:
+    """Sightings queued up to be written together. See :meth:`AsyncSightingDB.batch`."""
+
+    def __init__(
+        self,
+        client: AsyncSightingDB,
+        *,
+        chunk_size: int | None = None,
+        strict: bool = True,
+    ) -> None:
+        self._client = client
+        self._chunk_size = chunk_size
+        self._strict = strict
+        self._pending: list[Sighting] = []
+        self.result = BulkWriteResult(written=0)
+
+    async def add(
+        self,
+        namespace: str,
+        value: str,
+        *,
+        timestamp: Any = None,
+        ttl: Any = None,
+        noshadow: bool = False,
+    ) -> None:
+        """Queue one sighting, sending the batch if it is now full."""
+        self._pending.append(Sighting(namespace, value, timestamp, ttl, noshadow))
+        if self._chunk_size and len(self._pending) >= self._chunk_size:
+            await self.flush()
+
+    async def extend(
+        self, sightings: Iterable[Sighting | tuple[str, str] | dict[str, Any]]
+    ) -> None:
+        """Queue many sightings."""
+        for sighting in proto.coerce_sightings(sightings):
+            await self.add(
+                sighting.namespace,
+                sighting.value,
+                timestamp=sighting.timestamp,
+                ttl=sighting.ttl,
+                noshadow=sighting.noshadow,
+            )
+
+    async def flush(self) -> BulkWriteResult:
+        """Send whatever is queued, and return the total sent so far."""
+        if self._pending:
+            pending, self._pending = self._pending, []
+            self.result = self.result + await self._client.write_many(
+                pending, strict=self._strict
+            )
+        return self.result
+
+    def __len__(self) -> int:
+        return len(self._pending)
+
+    async def __aenter__(self) -> AsyncWriteBatch:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        if exc_type is None:
+            await self.flush()