qdrant-operator 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,124 @@
1
+ """Qdrant REST API adapter. Stateless: every call opens its own client against the given node."""
2
+
3
+ import ssl
4
+ from collections.abc import AsyncIterator
5
+ from contextlib import asynccontextmanager
6
+ from dataclasses import dataclass
7
+ from typing import Any
8
+
9
+ import httpx
10
+ from loguru import logger
11
+
12
+ from qdrant_operator.domain import JsonDict
13
+ from qdrant_operator.domain import QdrantNode
14
+ from qdrant_operator.domain import RestorePriority
15
+ from qdrant_operator.domain import Snapshot
16
+
17
+ STREAM_CHUNK_BYTES = 1024 * 1024
18
+
19
+
20
+ @dataclass
21
+ class QdrantAdapter:
22
+ timeout_seconds: float = 30.0
23
+ snapshot_timeout_seconds: float = 3600.0
24
+
25
+ @asynccontextmanager
26
+ async def client(self, node: QdrantNode, timeout: float) -> AsyncIterator[httpx.AsyncClient]:
27
+ async with httpx.AsyncClient(
28
+ base_url=node.url,
29
+ headers={"api-key": node.api_key} if node.api_key else {},
30
+ timeout=timeout,
31
+ verify=self.ssl_context(node),
32
+ ) as client:
33
+ yield client
34
+
35
+ @staticmethod
36
+ def ssl_context(node: QdrantNode) -> ssl.SSLContext | bool:
37
+ if not node.ca_cert:
38
+ return True
39
+ context = ssl.create_default_context()
40
+ context.load_verify_locations(cadata=node.ca_cert)
41
+ return context
42
+
43
+ async def list_collections(self, node: QdrantNode) -> list[str]:
44
+ async with self.client(node, self.timeout_seconds) as client:
45
+ result = await self.result(client.get("/collections"))
46
+ return [c["name"] for c in result["collections"]]
47
+
48
+ async def create_snapshot(self, node: QdrantNode, collection: str) -> Snapshot:
49
+ async with self.client(node, self.snapshot_timeout_seconds) as client:
50
+ result = await self.result(
51
+ client.post(f"/collections/{collection}/snapshots", params={"wait": "true"})
52
+ )
53
+ logger.info(
54
+ "snapshot created", node=node.url, collection=collection, snapshot=result["name"]
55
+ )
56
+ return Snapshot(
57
+ name=result["name"],
58
+ collection=collection,
59
+ size_bytes=result.get("size", 0),
60
+ checksum=result.get("checksum"),
61
+ )
62
+
63
+ async def stream_snapshot(
64
+ self, node: QdrantNode, collection: str, snapshot_name: str
65
+ ) -> AsyncIterator[bytes]:
66
+ path = f"/collections/{collection}/snapshots/{snapshot_name}"
67
+ async with (
68
+ self.client(node, self.snapshot_timeout_seconds) as client,
69
+ client.stream("GET", path) as response,
70
+ ):
71
+ response.raise_for_status()
72
+ async for chunk in response.aiter_bytes(STREAM_CHUNK_BYTES):
73
+ yield chunk
74
+
75
+ async def delete_snapshot(self, node: QdrantNode, collection: str, snapshot_name: str) -> None:
76
+ async with self.client(node, self.timeout_seconds) as client:
77
+ await self.result(
78
+ client.delete(
79
+ f"/collections/{collection}/snapshots/{snapshot_name}",
80
+ params={"wait": "true"},
81
+ )
82
+ )
83
+ logger.info(
84
+ "snapshot deleted", node=node.url, collection=collection, snapshot=snapshot_name
85
+ )
86
+
87
+ async def recover_snapshot(
88
+ self,
89
+ node: QdrantNode,
90
+ collection: str,
91
+ location: str,
92
+ priority: RestorePriority,
93
+ checksum: str | None,
94
+ ) -> None:
95
+ body: JsonDict = {"location": location, "priority": priority.value}
96
+ if checksum:
97
+ body["checksum"] = checksum
98
+ async with self.client(node, self.snapshot_timeout_seconds) as client:
99
+ await self.result(
100
+ client.put(
101
+ f"/collections/{collection}/snapshots/recover",
102
+ params={"wait": "true"},
103
+ json=body,
104
+ )
105
+ )
106
+ logger.info("snapshot recovered", node=node.url, collection=collection)
107
+
108
+ async def collection_info(self, node: QdrantNode, collection: str) -> JsonDict:
109
+ async with self.client(node, self.timeout_seconds) as client:
110
+ return await self.result(client.get(f"/collections/{collection}"))
111
+
112
+ async def ready(self, node: QdrantNode) -> bool:
113
+ try:
114
+ async with self.client(node, 5.0) as client:
115
+ response = await client.get("/readyz")
116
+ return response.status_code == 200
117
+ except httpx.HTTPError:
118
+ return False
119
+
120
+ @staticmethod
121
+ async def result(request: Any) -> JsonDict:
122
+ response: httpx.Response = await request
123
+ response.raise_for_status()
124
+ return response.json()["result"]
@@ -0,0 +1,113 @@
1
+ """S3-compatible object storage adapter built on aioboto3."""
2
+
3
+ from collections.abc import AsyncIterator
4
+ from contextlib import asynccontextmanager
5
+ from dataclasses import dataclass
6
+ from typing import Any
7
+
8
+ import aioboto3
9
+ from botocore.config import Config
10
+ from loguru import logger
11
+
12
+ from qdrant_operator.domain import S3Credentials
13
+ from qdrant_operator.domain import S3StorageSpec
14
+
15
+ DELETE_BATCH_SIZE = 1000
16
+
17
+
18
+ @dataclass
19
+ class ChunkReader:
20
+ """Adapts an async byte iterator to the async `read(n)` aioboto3 expects."""
21
+
22
+ chunks: AsyncIterator[bytes]
23
+ buffer: bytearray
24
+ exhausted: bool = False
25
+ total: int = 0
26
+
27
+ async def read(self, size: int) -> bytes:
28
+ while len(self.buffer) < size and not self.exhausted:
29
+ chunk = await anext(self.chunks, None)
30
+ if chunk is None:
31
+ self.exhausted = True
32
+ break
33
+ self.buffer.extend(chunk)
34
+ data = bytes(self.buffer[:size])
35
+ del self.buffer[:size]
36
+ self.total += len(data)
37
+ return data
38
+
39
+
40
+ @dataclass
41
+ class S3Adapter:
42
+ @asynccontextmanager
43
+ async def client(
44
+ self, storage: S3StorageSpec, credentials: S3Credentials
45
+ ) -> AsyncIterator[Any]:
46
+ session: Any = aioboto3.Session(
47
+ aws_access_key_id=credentials.access_key_id,
48
+ aws_secret_access_key=credentials.secret_access_key,
49
+ region_name=storage.region,
50
+ )
51
+ addressing = {"addressing_style": "path"} if storage.force_path_style else {}
52
+ async with session.client(
53
+ "s3",
54
+ endpoint_url=storage.endpoint,
55
+ config=Config(s3=addressing),
56
+ ) as client:
57
+ yield client
58
+
59
+ async def upload_stream(
60
+ self,
61
+ storage: S3StorageSpec,
62
+ credentials: S3Credentials,
63
+ key: str,
64
+ chunks: AsyncIterator[bytes],
65
+ ) -> int:
66
+ reader = ChunkReader(chunks, bytearray())
67
+ async with self.client(storage, credentials) as s3:
68
+ await s3.upload_fileobj(reader, storage.bucket, key)
69
+ logger.info("s3 upload complete", uri=storage.uri(key), bytes=reader.total)
70
+ return reader.total
71
+
72
+ async def put_object(
73
+ self, storage: S3StorageSpec, credentials: S3Credentials, key: str, data: bytes
74
+ ) -> None:
75
+ async with self.client(storage, credentials) as s3:
76
+ await s3.put_object(Bucket=storage.bucket, Key=key, Body=data)
77
+
78
+ async def get_object(
79
+ self, storage: S3StorageSpec, credentials: S3Credentials, key: str
80
+ ) -> bytes:
81
+ async with self.client(storage, credentials) as s3:
82
+ response = await s3.get_object(Bucket=storage.bucket, Key=key)
83
+ async with response["Body"] as body:
84
+ return await body.read()
85
+
86
+ async def delete_prefix(
87
+ self, storage: S3StorageSpec, credentials: S3Credentials, prefix: str
88
+ ) -> int:
89
+ deleted = 0
90
+ async with self.client(storage, credentials) as s3:
91
+ paginator = s3.get_paginator("list_objects_v2")
92
+ async for page in paginator.paginate(Bucket=storage.bucket, Prefix=f"{prefix}/"):
93
+ keys = [{"Key": item["Key"]} for item in page.get("Contents", [])]
94
+ for start in range(0, len(keys), DELETE_BATCH_SIZE):
95
+ batch = keys[start : start + DELETE_BATCH_SIZE]
96
+ await s3.delete_objects(Bucket=storage.bucket, Delete={"Objects": batch})
97
+ deleted += len(batch)
98
+ logger.info("s3 prefix deleted", uri=storage.uri(prefix), objects=deleted)
99
+ return deleted
100
+
101
+ async def presigned_get_url(
102
+ self,
103
+ storage: S3StorageSpec,
104
+ credentials: S3Credentials,
105
+ key: str,
106
+ expires_seconds: int,
107
+ ) -> str:
108
+ async with self.client(storage, credentials) as s3:
109
+ return await s3.generate_presigned_url(
110
+ "get_object",
111
+ Params={"Bucket": storage.bucket, "Key": key},
112
+ ExpiresIn=expires_seconds,
113
+ )