qdrant-operator 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qdrant_operator/__init__.py +1 -0
- qdrant_operator/container.py +49 -0
- qdrant_operator/domain.py +1245 -0
- qdrant_operator/handlers.py +189 -0
- qdrant_operator/helm_adapter.py +85 -0
- qdrant_operator/kubernetes_adapter.py +144 -0
- qdrant_operator/main.py +48 -0
- qdrant_operator/ports.py +116 -0
- qdrant_operator/py.typed +0 -0
- qdrant_operator/qdrant_adapter.py +124 -0
- qdrant_operator/s3_adapter.py +113 -0
- qdrant_operator/usecases.py +597 -0
- qdrant_operator-0.2.0.dist-info/METADATA +284 -0
- qdrant_operator-0.2.0.dist-info/RECORD +17 -0
- qdrant_operator-0.2.0.dist-info/WHEEL +4 -0
- qdrant_operator-0.2.0.dist-info/entry_points.txt +3 -0
- qdrant_operator-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Qdrant REST API adapter. Stateless: every call opens its own client against the given node."""
|
|
2
|
+
|
|
3
|
+
import ssl
|
|
4
|
+
from collections.abc import AsyncIterator
|
|
5
|
+
from contextlib import asynccontextmanager
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
from loguru import logger
|
|
11
|
+
|
|
12
|
+
from qdrant_operator.domain import JsonDict
|
|
13
|
+
from qdrant_operator.domain import QdrantNode
|
|
14
|
+
from qdrant_operator.domain import RestorePriority
|
|
15
|
+
from qdrant_operator.domain import Snapshot
|
|
16
|
+
|
|
17
|
+
STREAM_CHUNK_BYTES = 1024 * 1024
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class QdrantAdapter:
|
|
22
|
+
timeout_seconds: float = 30.0
|
|
23
|
+
snapshot_timeout_seconds: float = 3600.0
|
|
24
|
+
|
|
25
|
+
@asynccontextmanager
|
|
26
|
+
async def client(self, node: QdrantNode, timeout: float) -> AsyncIterator[httpx.AsyncClient]:
|
|
27
|
+
async with httpx.AsyncClient(
|
|
28
|
+
base_url=node.url,
|
|
29
|
+
headers={"api-key": node.api_key} if node.api_key else {},
|
|
30
|
+
timeout=timeout,
|
|
31
|
+
verify=self.ssl_context(node),
|
|
32
|
+
) as client:
|
|
33
|
+
yield client
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def ssl_context(node: QdrantNode) -> ssl.SSLContext | bool:
|
|
37
|
+
if not node.ca_cert:
|
|
38
|
+
return True
|
|
39
|
+
context = ssl.create_default_context()
|
|
40
|
+
context.load_verify_locations(cadata=node.ca_cert)
|
|
41
|
+
return context
|
|
42
|
+
|
|
43
|
+
async def list_collections(self, node: QdrantNode) -> list[str]:
|
|
44
|
+
async with self.client(node, self.timeout_seconds) as client:
|
|
45
|
+
result = await self.result(client.get("/collections"))
|
|
46
|
+
return [c["name"] for c in result["collections"]]
|
|
47
|
+
|
|
48
|
+
async def create_snapshot(self, node: QdrantNode, collection: str) -> Snapshot:
|
|
49
|
+
async with self.client(node, self.snapshot_timeout_seconds) as client:
|
|
50
|
+
result = await self.result(
|
|
51
|
+
client.post(f"/collections/{collection}/snapshots", params={"wait": "true"})
|
|
52
|
+
)
|
|
53
|
+
logger.info(
|
|
54
|
+
"snapshot created", node=node.url, collection=collection, snapshot=result["name"]
|
|
55
|
+
)
|
|
56
|
+
return Snapshot(
|
|
57
|
+
name=result["name"],
|
|
58
|
+
collection=collection,
|
|
59
|
+
size_bytes=result.get("size", 0),
|
|
60
|
+
checksum=result.get("checksum"),
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
async def stream_snapshot(
|
|
64
|
+
self, node: QdrantNode, collection: str, snapshot_name: str
|
|
65
|
+
) -> AsyncIterator[bytes]:
|
|
66
|
+
path = f"/collections/{collection}/snapshots/{snapshot_name}"
|
|
67
|
+
async with (
|
|
68
|
+
self.client(node, self.snapshot_timeout_seconds) as client,
|
|
69
|
+
client.stream("GET", path) as response,
|
|
70
|
+
):
|
|
71
|
+
response.raise_for_status()
|
|
72
|
+
async for chunk in response.aiter_bytes(STREAM_CHUNK_BYTES):
|
|
73
|
+
yield chunk
|
|
74
|
+
|
|
75
|
+
async def delete_snapshot(self, node: QdrantNode, collection: str, snapshot_name: str) -> None:
|
|
76
|
+
async with self.client(node, self.timeout_seconds) as client:
|
|
77
|
+
await self.result(
|
|
78
|
+
client.delete(
|
|
79
|
+
f"/collections/{collection}/snapshots/{snapshot_name}",
|
|
80
|
+
params={"wait": "true"},
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
logger.info(
|
|
84
|
+
"snapshot deleted", node=node.url, collection=collection, snapshot=snapshot_name
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
async def recover_snapshot(
|
|
88
|
+
self,
|
|
89
|
+
node: QdrantNode,
|
|
90
|
+
collection: str,
|
|
91
|
+
location: str,
|
|
92
|
+
priority: RestorePriority,
|
|
93
|
+
checksum: str | None,
|
|
94
|
+
) -> None:
|
|
95
|
+
body: JsonDict = {"location": location, "priority": priority.value}
|
|
96
|
+
if checksum:
|
|
97
|
+
body["checksum"] = checksum
|
|
98
|
+
async with self.client(node, self.snapshot_timeout_seconds) as client:
|
|
99
|
+
await self.result(
|
|
100
|
+
client.put(
|
|
101
|
+
f"/collections/{collection}/snapshots/recover",
|
|
102
|
+
params={"wait": "true"},
|
|
103
|
+
json=body,
|
|
104
|
+
)
|
|
105
|
+
)
|
|
106
|
+
logger.info("snapshot recovered", node=node.url, collection=collection)
|
|
107
|
+
|
|
108
|
+
async def collection_info(self, node: QdrantNode, collection: str) -> JsonDict:
|
|
109
|
+
async with self.client(node, self.timeout_seconds) as client:
|
|
110
|
+
return await self.result(client.get(f"/collections/{collection}"))
|
|
111
|
+
|
|
112
|
+
async def ready(self, node: QdrantNode) -> bool:
|
|
113
|
+
try:
|
|
114
|
+
async with self.client(node, 5.0) as client:
|
|
115
|
+
response = await client.get("/readyz")
|
|
116
|
+
return response.status_code == 200
|
|
117
|
+
except httpx.HTTPError:
|
|
118
|
+
return False
|
|
119
|
+
|
|
120
|
+
@staticmethod
|
|
121
|
+
async def result(request: Any) -> JsonDict:
|
|
122
|
+
response: httpx.Response = await request
|
|
123
|
+
response.raise_for_status()
|
|
124
|
+
return response.json()["result"]
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""S3-compatible object storage adapter built on aioboto3."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncIterator
|
|
4
|
+
from contextlib import asynccontextmanager
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import aioboto3
|
|
9
|
+
from botocore.config import Config
|
|
10
|
+
from loguru import logger
|
|
11
|
+
|
|
12
|
+
from qdrant_operator.domain import S3Credentials
|
|
13
|
+
from qdrant_operator.domain import S3StorageSpec
|
|
14
|
+
|
|
15
|
+
DELETE_BATCH_SIZE = 1000
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class ChunkReader:
|
|
20
|
+
"""Adapts an async byte iterator to the async `read(n)` aioboto3 expects."""
|
|
21
|
+
|
|
22
|
+
chunks: AsyncIterator[bytes]
|
|
23
|
+
buffer: bytearray
|
|
24
|
+
exhausted: bool = False
|
|
25
|
+
total: int = 0
|
|
26
|
+
|
|
27
|
+
async def read(self, size: int) -> bytes:
|
|
28
|
+
while len(self.buffer) < size and not self.exhausted:
|
|
29
|
+
chunk = await anext(self.chunks, None)
|
|
30
|
+
if chunk is None:
|
|
31
|
+
self.exhausted = True
|
|
32
|
+
break
|
|
33
|
+
self.buffer.extend(chunk)
|
|
34
|
+
data = bytes(self.buffer[:size])
|
|
35
|
+
del self.buffer[:size]
|
|
36
|
+
self.total += len(data)
|
|
37
|
+
return data
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class S3Adapter:
|
|
42
|
+
@asynccontextmanager
|
|
43
|
+
async def client(
|
|
44
|
+
self, storage: S3StorageSpec, credentials: S3Credentials
|
|
45
|
+
) -> AsyncIterator[Any]:
|
|
46
|
+
session: Any = aioboto3.Session(
|
|
47
|
+
aws_access_key_id=credentials.access_key_id,
|
|
48
|
+
aws_secret_access_key=credentials.secret_access_key,
|
|
49
|
+
region_name=storage.region,
|
|
50
|
+
)
|
|
51
|
+
addressing = {"addressing_style": "path"} if storage.force_path_style else {}
|
|
52
|
+
async with session.client(
|
|
53
|
+
"s3",
|
|
54
|
+
endpoint_url=storage.endpoint,
|
|
55
|
+
config=Config(s3=addressing),
|
|
56
|
+
) as client:
|
|
57
|
+
yield client
|
|
58
|
+
|
|
59
|
+
async def upload_stream(
|
|
60
|
+
self,
|
|
61
|
+
storage: S3StorageSpec,
|
|
62
|
+
credentials: S3Credentials,
|
|
63
|
+
key: str,
|
|
64
|
+
chunks: AsyncIterator[bytes],
|
|
65
|
+
) -> int:
|
|
66
|
+
reader = ChunkReader(chunks, bytearray())
|
|
67
|
+
async with self.client(storage, credentials) as s3:
|
|
68
|
+
await s3.upload_fileobj(reader, storage.bucket, key)
|
|
69
|
+
logger.info("s3 upload complete", uri=storage.uri(key), bytes=reader.total)
|
|
70
|
+
return reader.total
|
|
71
|
+
|
|
72
|
+
async def put_object(
|
|
73
|
+
self, storage: S3StorageSpec, credentials: S3Credentials, key: str, data: bytes
|
|
74
|
+
) -> None:
|
|
75
|
+
async with self.client(storage, credentials) as s3:
|
|
76
|
+
await s3.put_object(Bucket=storage.bucket, Key=key, Body=data)
|
|
77
|
+
|
|
78
|
+
async def get_object(
|
|
79
|
+
self, storage: S3StorageSpec, credentials: S3Credentials, key: str
|
|
80
|
+
) -> bytes:
|
|
81
|
+
async with self.client(storage, credentials) as s3:
|
|
82
|
+
response = await s3.get_object(Bucket=storage.bucket, Key=key)
|
|
83
|
+
async with response["Body"] as body:
|
|
84
|
+
return await body.read()
|
|
85
|
+
|
|
86
|
+
async def delete_prefix(
|
|
87
|
+
self, storage: S3StorageSpec, credentials: S3Credentials, prefix: str
|
|
88
|
+
) -> int:
|
|
89
|
+
deleted = 0
|
|
90
|
+
async with self.client(storage, credentials) as s3:
|
|
91
|
+
paginator = s3.get_paginator("list_objects_v2")
|
|
92
|
+
async for page in paginator.paginate(Bucket=storage.bucket, Prefix=f"{prefix}/"):
|
|
93
|
+
keys = [{"Key": item["Key"]} for item in page.get("Contents", [])]
|
|
94
|
+
for start in range(0, len(keys), DELETE_BATCH_SIZE):
|
|
95
|
+
batch = keys[start : start + DELETE_BATCH_SIZE]
|
|
96
|
+
await s3.delete_objects(Bucket=storage.bucket, Delete={"Objects": batch})
|
|
97
|
+
deleted += len(batch)
|
|
98
|
+
logger.info("s3 prefix deleted", uri=storage.uri(prefix), objects=deleted)
|
|
99
|
+
return deleted
|
|
100
|
+
|
|
101
|
+
async def presigned_get_url(
|
|
102
|
+
self,
|
|
103
|
+
storage: S3StorageSpec,
|
|
104
|
+
credentials: S3Credentials,
|
|
105
|
+
key: str,
|
|
106
|
+
expires_seconds: int,
|
|
107
|
+
) -> str:
|
|
108
|
+
async with self.client(storage, credentials) as s3:
|
|
109
|
+
return await s3.generate_presigned_url(
|
|
110
|
+
"get_object",
|
|
111
|
+
Params={"Bucket": storage.bucket, "Key": key},
|
|
112
|
+
ExpiresIn=expires_seconds,
|
|
113
|
+
)
|