qdrant-operator 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qdrant_operator/__init__.py +1 -0
- qdrant_operator/container.py +49 -0
- qdrant_operator/domain.py +1245 -0
- qdrant_operator/handlers.py +189 -0
- qdrant_operator/helm_adapter.py +85 -0
- qdrant_operator/kubernetes_adapter.py +144 -0
- qdrant_operator/main.py +48 -0
- qdrant_operator/ports.py +116 -0
- qdrant_operator/py.typed +0 -0
- qdrant_operator/qdrant_adapter.py +124 -0
- qdrant_operator/s3_adapter.py +113 -0
- qdrant_operator/usecases.py +597 -0
- qdrant_operator-0.2.0.dist-info/METADATA +284 -0
- qdrant_operator-0.2.0.dist-info/RECORD +17 -0
- qdrant_operator-0.2.0.dist-info/WHEEL +4 -0
- qdrant_operator-0.2.0.dist-info/entry_points.txt +3 -0
- qdrant_operator-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
"""Application use cases: orchestrate ports, never touch I/O libraries directly."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from dataclasses import replace
|
|
8
|
+
from datetime import UTC
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from qdrant_operator.domain import BACKUPS
|
|
13
|
+
from qdrant_operator.domain import MANIFEST_KEY
|
|
14
|
+
from qdrant_operator.domain import BackupManifest
|
|
15
|
+
from qdrant_operator.domain import BackupPhase
|
|
16
|
+
from qdrant_operator.domain import BackupRecord
|
|
17
|
+
from qdrant_operator.domain import BackupScheduleSpec
|
|
18
|
+
from qdrant_operator.domain import BackupScheduleStatus
|
|
19
|
+
from qdrant_operator.domain import BackupSpec
|
|
20
|
+
from qdrant_operator.domain import BackupStatus
|
|
21
|
+
from qdrant_operator.domain import ClusterConnection
|
|
22
|
+
from qdrant_operator.domain import ClusterPhase
|
|
23
|
+
from qdrant_operator.domain import ClusterRef
|
|
24
|
+
from qdrant_operator.domain import ClusterSpec
|
|
25
|
+
from qdrant_operator.domain import ClusterStatus
|
|
26
|
+
from qdrant_operator.domain import CollectionBackup
|
|
27
|
+
from qdrant_operator.domain import CollectionBackupStatus
|
|
28
|
+
from qdrant_operator.domain import ConcurrencyPolicy
|
|
29
|
+
from qdrant_operator.domain import Condition
|
|
30
|
+
from qdrant_operator.domain import ConditionStatus
|
|
31
|
+
from qdrant_operator.domain import CredentialsSecretRef
|
|
32
|
+
from qdrant_operator.domain import QdrantNode
|
|
33
|
+
from qdrant_operator.domain import ResourceRef
|
|
34
|
+
from qdrant_operator.domain import RestoredCollection
|
|
35
|
+
from qdrant_operator.domain import RestorePhase
|
|
36
|
+
from qdrant_operator.domain import RestoreProgress
|
|
37
|
+
from qdrant_operator.domain import RestoreSpec
|
|
38
|
+
from qdrant_operator.domain import RestoreStatus
|
|
39
|
+
from qdrant_operator.domain import S3Credentials
|
|
40
|
+
from qdrant_operator.domain import S3StorageSpec
|
|
41
|
+
from qdrant_operator.domain import SchedulePhase
|
|
42
|
+
from qdrant_operator.domain import SnapshotRecord
|
|
43
|
+
from qdrant_operator.domain import chart_version
|
|
44
|
+
from qdrant_operator.domain import format_size
|
|
45
|
+
from qdrant_operator.domain import join_key
|
|
46
|
+
from qdrant_operator.domain import parse_time
|
|
47
|
+
from qdrant_operator.domain import set_condition
|
|
48
|
+
from qdrant_operator.ports import HelmPort
|
|
49
|
+
from qdrant_operator.ports import KubernetesPort
|
|
50
|
+
from qdrant_operator.ports import QdrantPort
|
|
51
|
+
from qdrant_operator.ports import StoragePort
|
|
52
|
+
|
|
53
|
+
RECENT_BACKUPS_LIMIT = 10
|
|
54
|
+
PRESIGNED_URL_TTL_SECONDS = 3600
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ClusterNotFoundError(LookupError):
|
|
58
|
+
"""The referenced QdrantCluster does not exist; callers retry rather than fail."""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SourceNotReadyError(RuntimeError):
|
|
62
|
+
"""The referenced QdrantBackup has not completed yet; callers retry rather than fail."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def now_utc() -> datetime:
|
|
66
|
+
return datetime.now(UTC)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
async def read_credentials(kubernetes: KubernetesPort, ref: CredentialsSecretRef) -> S3Credentials:
|
|
70
|
+
return S3Credentials(
|
|
71
|
+
access_key_id=await kubernetes.get_secret_value(ref.access_key_ref()),
|
|
72
|
+
secret_access_key=await kubernetes.get_secret_value(ref.secret_key_ref()),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass
|
|
77
|
+
class ResolveCluster:
|
|
78
|
+
kubernetes: KubernetesPort
|
|
79
|
+
|
|
80
|
+
async def execute(self, ref: ClusterRef) -> ClusterConnection:
|
|
81
|
+
body = await self.kubernetes.get_custom_resource(ref.to_resource_ref())
|
|
82
|
+
if not body:
|
|
83
|
+
raise ClusterNotFoundError(f"QdrantCluster {ref.namespace}/{ref.name} not found")
|
|
84
|
+
spec = ClusterSpec.from_dict(body["spec"], body["metadata"])
|
|
85
|
+
api_key = await self.read_api_key(spec)
|
|
86
|
+
ca_cert = await self.read_ca_cert(spec)
|
|
87
|
+
return ClusterConnection(
|
|
88
|
+
service=QdrantNode(spec.service_url(), api_key, ca_cert),
|
|
89
|
+
nodes=tuple(QdrantNode(url, api_key, ca_cert) for url in spec.node_urls()),
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
async def read_api_key(self, spec: ClusterSpec) -> str | None:
|
|
93
|
+
secret_ref = spec.api_key.resolve_secret_ref(spec.name, spec.namespace)
|
|
94
|
+
if not secret_ref:
|
|
95
|
+
return None
|
|
96
|
+
return await self.kubernetes.get_secret_value(secret_ref)
|
|
97
|
+
|
|
98
|
+
async def read_ca_cert(self, spec: ClusterSpec) -> str | None:
|
|
99
|
+
secret_ref = spec.tls.ca_secret_ref(spec.namespace)
|
|
100
|
+
if not secret_ref:
|
|
101
|
+
return None
|
|
102
|
+
try:
|
|
103
|
+
return await self.kubernetes.get_secret_value(secret_ref)
|
|
104
|
+
except KeyError:
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
@dataclass
|
|
109
|
+
class ReconcileCluster:
|
|
110
|
+
helm: HelmPort
|
|
111
|
+
|
|
112
|
+
async def execute(
|
|
113
|
+
self, spec: ClusterSpec, generation: int | None, current: ClusterStatus
|
|
114
|
+
) -> ClusterStatus:
|
|
115
|
+
exists = await self.helm.release_exists(spec.release_name, spec.namespace)
|
|
116
|
+
await self.helm.apply(
|
|
117
|
+
release_name=spec.release_name,
|
|
118
|
+
namespace=spec.namespace,
|
|
119
|
+
chart_version=chart_version(spec.version),
|
|
120
|
+
values=spec.to_helm_values(),
|
|
121
|
+
)
|
|
122
|
+
phase = ClusterPhase.UPGRADING if exists else ClusterPhase.PENDING
|
|
123
|
+
conditions = set_condition(
|
|
124
|
+
current.conditions,
|
|
125
|
+
Condition(
|
|
126
|
+
type="Progressing",
|
|
127
|
+
status=ConditionStatus.TRUE,
|
|
128
|
+
last_transition_time=now_utc(),
|
|
129
|
+
reason="HelmReleaseApplied",
|
|
130
|
+
message=f"Helm release {spec.release_name} applied",
|
|
131
|
+
),
|
|
132
|
+
)
|
|
133
|
+
return replace(
|
|
134
|
+
current,
|
|
135
|
+
phase=phase,
|
|
136
|
+
replicas=spec.replicas,
|
|
137
|
+
helm_release=spec.release_name,
|
|
138
|
+
endpoint=spec.service_url(),
|
|
139
|
+
version=spec.version,
|
|
140
|
+
conditions=tuple(conditions),
|
|
141
|
+
observed_generation=generation,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass
|
|
146
|
+
class ObserveCluster:
|
|
147
|
+
"""Lift StatefulSet readiness into the QdrantCluster status."""
|
|
148
|
+
|
|
149
|
+
kubernetes: KubernetesPort
|
|
150
|
+
|
|
151
|
+
async def execute(self, spec: ClusterSpec, current: ClusterStatus) -> ClusterStatus:
|
|
152
|
+
statefulset = await self.kubernetes.get_statefulset_status(
|
|
153
|
+
spec.release_name, spec.namespace
|
|
154
|
+
)
|
|
155
|
+
ready_replicas = statefulset.ready_replicas if statefulset else 0
|
|
156
|
+
ready = ready_replicas >= spec.replicas
|
|
157
|
+
phase = ClusterPhase.RUNNING if ready else self.pending_phase(current)
|
|
158
|
+
conditions = set_condition(
|
|
159
|
+
current.conditions,
|
|
160
|
+
Condition(
|
|
161
|
+
type="Ready",
|
|
162
|
+
status=ConditionStatus.TRUE if ready else ConditionStatus.FALSE,
|
|
163
|
+
last_transition_time=now_utc(),
|
|
164
|
+
reason="AllReplicasReady" if ready else "ReplicasNotReady",
|
|
165
|
+
message=f"{ready_replicas}/{spec.replicas} replicas ready",
|
|
166
|
+
),
|
|
167
|
+
)
|
|
168
|
+
if ready:
|
|
169
|
+
conditions = set_condition(
|
|
170
|
+
conditions,
|
|
171
|
+
Condition(
|
|
172
|
+
type="Progressing",
|
|
173
|
+
status=ConditionStatus.FALSE,
|
|
174
|
+
last_transition_time=now_utc(),
|
|
175
|
+
reason="RolloutComplete",
|
|
176
|
+
message="StatefulSet rollout complete",
|
|
177
|
+
),
|
|
178
|
+
)
|
|
179
|
+
return replace(
|
|
180
|
+
current,
|
|
181
|
+
phase=phase,
|
|
182
|
+
replicas=spec.replicas,
|
|
183
|
+
ready_replicas=ready_replicas,
|
|
184
|
+
endpoint=spec.service_url(),
|
|
185
|
+
conditions=tuple(conditions),
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
@staticmethod
|
|
189
|
+
def pending_phase(current: ClusterStatus) -> ClusterPhase:
|
|
190
|
+
if current.phase in (ClusterPhase.UPGRADING, ClusterPhase.FAILED):
|
|
191
|
+
return current.phase
|
|
192
|
+
return ClusterPhase.PENDING
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass
|
|
196
|
+
class DeleteCluster:
|
|
197
|
+
helm: HelmPort
|
|
198
|
+
|
|
199
|
+
async def execute(self, spec: ClusterSpec) -> None:
|
|
200
|
+
if not await self.helm.release_exists(spec.release_name, spec.namespace):
|
|
201
|
+
return
|
|
202
|
+
await self.helm.uninstall(spec.release_name, spec.namespace)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@dataclass
|
|
206
|
+
class ExecuteBackup:
|
|
207
|
+
qdrant: QdrantPort
|
|
208
|
+
storage: StoragePort
|
|
209
|
+
kubernetes: KubernetesPort
|
|
210
|
+
|
|
211
|
+
async def execute(self, spec: BackupSpec, ref: ResourceRef) -> BackupStatus:
|
|
212
|
+
start_time = now_utc()
|
|
213
|
+
connection = await ResolveCluster(self.kubernetes).execute(spec.cluster_ref)
|
|
214
|
+
credentials = await read_credentials(self.kubernetes, spec.storage.credentials_secret_ref)
|
|
215
|
+
await self.kubernetes.patch_status(
|
|
216
|
+
ref, BackupStatus(phase=BackupPhase.IN_PROGRESS, start_time=start_time).to_dict()
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
collections = spec.collections or tuple(
|
|
220
|
+
await self.qdrant.list_collections(connection.service)
|
|
221
|
+
)
|
|
222
|
+
statuses = [
|
|
223
|
+
await self.backup_collection(spec, credentials, connection, collection)
|
|
224
|
+
for collection in collections
|
|
225
|
+
]
|
|
226
|
+
completed = [s for s in statuses if s.status == BackupPhase.COMPLETED]
|
|
227
|
+
failed = [s for s in statuses if s.status == BackupPhase.FAILED]
|
|
228
|
+
|
|
229
|
+
manifest = BackupManifest(
|
|
230
|
+
backup_name=spec.name,
|
|
231
|
+
cluster=spec.cluster_ref,
|
|
232
|
+
node_count=len(connection.nodes),
|
|
233
|
+
created_at=start_time,
|
|
234
|
+
collections=tuple(CollectionBackup(s.name, s.snapshots) for s in completed),
|
|
235
|
+
)
|
|
236
|
+
await self.storage.put_object(
|
|
237
|
+
spec.storage,
|
|
238
|
+
credentials,
|
|
239
|
+
join_key(spec.root_key, MANIFEST_KEY),
|
|
240
|
+
json.dumps(manifest.to_dict()).encode(),
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
completion_time = now_utc()
|
|
244
|
+
phase = BackupPhase.FAILED if failed else BackupPhase.COMPLETED
|
|
245
|
+
return BackupStatus(
|
|
246
|
+
phase=phase,
|
|
247
|
+
start_time=start_time,
|
|
248
|
+
completion_time=completion_time,
|
|
249
|
+
expires_at=spec.expires_at(completion_time),
|
|
250
|
+
s3_path=spec.storage.uri(spec.name),
|
|
251
|
+
total_size=format_size(manifest.size_bytes),
|
|
252
|
+
collections=tuple(statuses),
|
|
253
|
+
error=failed[0].error if failed else None,
|
|
254
|
+
conditions=(
|
|
255
|
+
Condition(
|
|
256
|
+
type="Complete",
|
|
257
|
+
status=ConditionStatus.FALSE if failed else ConditionStatus.TRUE,
|
|
258
|
+
last_transition_time=completion_time,
|
|
259
|
+
reason="BackupFailed" if failed else "BackupCompleted",
|
|
260
|
+
message=f"Backed up {len(completed)}/{len(collections)} collections",
|
|
261
|
+
),
|
|
262
|
+
),
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
async def backup_collection(
|
|
266
|
+
self,
|
|
267
|
+
spec: BackupSpec,
|
|
268
|
+
credentials: S3Credentials,
|
|
269
|
+
connection: ClusterConnection,
|
|
270
|
+
collection: str,
|
|
271
|
+
) -> CollectionBackupStatus:
|
|
272
|
+
try:
|
|
273
|
+
records = [
|
|
274
|
+
await self.backup_node(spec, credentials, node, index, collection)
|
|
275
|
+
for index, node in enumerate(connection.nodes)
|
|
276
|
+
]
|
|
277
|
+
except Exception as error:
|
|
278
|
+
return CollectionBackupStatus(
|
|
279
|
+
name=collection, status=BackupPhase.FAILED, error=str(error)
|
|
280
|
+
)
|
|
281
|
+
return CollectionBackupStatus(
|
|
282
|
+
name=collection, status=BackupPhase.COMPLETED, snapshots=tuple(records)
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
async def backup_node(
|
|
286
|
+
self,
|
|
287
|
+
spec: BackupSpec,
|
|
288
|
+
credentials: S3Credentials,
|
|
289
|
+
node: QdrantNode,
|
|
290
|
+
index: int,
|
|
291
|
+
collection: str,
|
|
292
|
+
) -> SnapshotRecord:
|
|
293
|
+
"""Snapshot one node's shards of a collection and stream them straight into the bucket."""
|
|
294
|
+
snapshot = await self.qdrant.create_snapshot(node, collection)
|
|
295
|
+
key = spec.snapshot_key(collection, index, snapshot.name)
|
|
296
|
+
try:
|
|
297
|
+
size = await self.storage.upload_stream(
|
|
298
|
+
spec.storage,
|
|
299
|
+
credentials,
|
|
300
|
+
key,
|
|
301
|
+
self.qdrant.stream_snapshot(node, collection, snapshot.name),
|
|
302
|
+
)
|
|
303
|
+
finally:
|
|
304
|
+
await self.qdrant.delete_snapshot(node, collection, snapshot.name)
|
|
305
|
+
return SnapshotRecord(
|
|
306
|
+
node_index=index,
|
|
307
|
+
key=key,
|
|
308
|
+
snapshot_name=snapshot.name,
|
|
309
|
+
size_bytes=size,
|
|
310
|
+
checksum=snapshot.checksum,
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@dataclass
|
|
315
|
+
class DeleteBackupData:
|
|
316
|
+
storage: StoragePort
|
|
317
|
+
kubernetes: KubernetesPort
|
|
318
|
+
|
|
319
|
+
async def execute(self, spec: BackupSpec) -> int:
|
|
320
|
+
credentials = await read_credentials(self.kubernetes, spec.storage.credentials_secret_ref)
|
|
321
|
+
return await self.storage.delete_prefix(spec.storage, credentials, spec.root_key)
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
@dataclass
|
|
325
|
+
class ExpireBackup:
|
|
326
|
+
"""Delete a QdrantBackup once its retentionDays have passed."""
|
|
327
|
+
|
|
328
|
+
kubernetes: KubernetesPort
|
|
329
|
+
|
|
330
|
+
async def execute(self, ref: ResourceRef, status: BackupStatus, now: datetime) -> bool:
|
|
331
|
+
if not status.expires_at or status.expires_at > now:
|
|
332
|
+
return False
|
|
333
|
+
await self.kubernetes.delete_custom_resource(ref)
|
|
334
|
+
return True
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
@dataclass(frozen=True)
|
|
338
|
+
class RestoreSource:
|
|
339
|
+
storage: S3StorageSpec
|
|
340
|
+
root_key: str
|
|
341
|
+
description: str
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
@dataclass
|
|
345
|
+
class ExecuteRestore:
|
|
346
|
+
qdrant: QdrantPort
|
|
347
|
+
storage: StoragePort
|
|
348
|
+
kubernetes: KubernetesPort
|
|
349
|
+
indexing_poll_seconds: float = 5.0
|
|
350
|
+
indexing_timeout_seconds: float = 3600.0
|
|
351
|
+
|
|
352
|
+
async def execute(self, spec: RestoreSpec, ref: ResourceRef) -> RestoreStatus:
|
|
353
|
+
start_time = now_utc()
|
|
354
|
+
source = await self.resolve_source(spec)
|
|
355
|
+
credentials = await read_credentials(self.kubernetes, source.storage.credentials_secret_ref)
|
|
356
|
+
connection = await ResolveCluster(self.kubernetes).execute(spec.target_cluster_ref)
|
|
357
|
+
|
|
358
|
+
await self.kubernetes.patch_status(
|
|
359
|
+
ref,
|
|
360
|
+
RestoreStatus(
|
|
361
|
+
phase=RestorePhase.DOWNLOADING,
|
|
362
|
+
start_time=start_time,
|
|
363
|
+
source_backup=source.description,
|
|
364
|
+
).to_dict(),
|
|
365
|
+
)
|
|
366
|
+
manifest = BackupManifest.from_dict(
|
|
367
|
+
json.loads(
|
|
368
|
+
await self.storage.get_object(
|
|
369
|
+
source.storage, credentials, join_key(source.root_key, MANIFEST_KEY)
|
|
370
|
+
)
|
|
371
|
+
)
|
|
372
|
+
)
|
|
373
|
+
if manifest.node_count != len(connection.nodes):
|
|
374
|
+
raise ValueError(
|
|
375
|
+
f"Backup was taken from {manifest.node_count} nodes but target cluster has "
|
|
376
|
+
f"{len(connection.nodes)}; node counts must match"
|
|
377
|
+
)
|
|
378
|
+
collections = spec.select_collections(manifest.collection_names)
|
|
379
|
+
|
|
380
|
+
restored: list[RestoredCollection] = []
|
|
381
|
+
for index, collection in enumerate(collections):
|
|
382
|
+
await self.kubernetes.patch_status(
|
|
383
|
+
ref,
|
|
384
|
+
RestoreStatus(
|
|
385
|
+
phase=RestorePhase.RESTORING,
|
|
386
|
+
start_time=start_time,
|
|
387
|
+
source_backup=source.description,
|
|
388
|
+
restored_collections=tuple(restored),
|
|
389
|
+
progress=RestoreProgress(len(collections), index, collection),
|
|
390
|
+
).to_dict(),
|
|
391
|
+
)
|
|
392
|
+
restored.append(
|
|
393
|
+
await self.restore_collection(
|
|
394
|
+
spec, source, credentials, connection, manifest, collection
|
|
395
|
+
)
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
failed = [r for r in restored if r.status == RestorePhase.FAILED]
|
|
399
|
+
completion_time = now_utc()
|
|
400
|
+
return RestoreStatus(
|
|
401
|
+
phase=RestorePhase.FAILED if failed else RestorePhase.COMPLETED,
|
|
402
|
+
start_time=start_time,
|
|
403
|
+
completion_time=completion_time,
|
|
404
|
+
source_backup=source.description,
|
|
405
|
+
restored_collections=tuple(restored),
|
|
406
|
+
progress=RestoreProgress(len(collections), len(collections) - len(failed)),
|
|
407
|
+
error=failed[0].error if failed else None,
|
|
408
|
+
conditions=(
|
|
409
|
+
Condition(
|
|
410
|
+
type="Complete",
|
|
411
|
+
status=ConditionStatus.FALSE if failed else ConditionStatus.TRUE,
|
|
412
|
+
last_transition_time=completion_time,
|
|
413
|
+
reason="RestoreFailed" if failed else "RestoreCompleted",
|
|
414
|
+
message=f"Restored {len(restored) - len(failed)}/{len(collections)}",
|
|
415
|
+
),
|
|
416
|
+
),
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
async def resolve_source(self, spec: RestoreSpec) -> RestoreSource:
|
|
420
|
+
if spec.source_s3:
|
|
421
|
+
return RestoreSource(
|
|
422
|
+
storage=spec.source_s3,
|
|
423
|
+
root_key=spec.source_s3.prefix,
|
|
424
|
+
description=spec.source_s3.uri(),
|
|
425
|
+
)
|
|
426
|
+
if not spec.backup_ref:
|
|
427
|
+
raise ValueError("QdrantRestore needs either spec.backupRef or spec.source.s3")
|
|
428
|
+
body = await self.kubernetes.get_custom_resource(spec.backup_ref.to_resource_ref())
|
|
429
|
+
if not body:
|
|
430
|
+
raise LookupError(
|
|
431
|
+
f"QdrantBackup {spec.backup_ref.namespace}/{spec.backup_ref.name} not found"
|
|
432
|
+
)
|
|
433
|
+
status = BackupStatus.from_dict(body.get("status", {}))
|
|
434
|
+
if status.phase != BackupPhase.COMPLETED:
|
|
435
|
+
raise SourceNotReadyError(
|
|
436
|
+
f"QdrantBackup {spec.backup_ref.name} is {status.phase.value}, not Completed"
|
|
437
|
+
)
|
|
438
|
+
backup = BackupSpec.from_dict(body["spec"], body["metadata"])
|
|
439
|
+
return RestoreSource(
|
|
440
|
+
storage=backup.storage, root_key=backup.root_key, description=spec.backup_ref.name
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
async def restore_collection(
|
|
444
|
+
self,
|
|
445
|
+
spec: RestoreSpec,
|
|
446
|
+
source: RestoreSource,
|
|
447
|
+
credentials: S3Credentials,
|
|
448
|
+
connection: ClusterConnection,
|
|
449
|
+
manifest: BackupManifest,
|
|
450
|
+
collection: str,
|
|
451
|
+
) -> RestoredCollection:
|
|
452
|
+
target = spec.target_name(collection)
|
|
453
|
+
original = collection if collection != target else None
|
|
454
|
+
backup = manifest.collection(collection)
|
|
455
|
+
if not backup:
|
|
456
|
+
return RestoredCollection(
|
|
457
|
+
name=target,
|
|
458
|
+
original_name=original,
|
|
459
|
+
status=RestorePhase.FAILED,
|
|
460
|
+
error=f"Collection {collection} missing from manifest",
|
|
461
|
+
)
|
|
462
|
+
try:
|
|
463
|
+
for record in backup.snapshots:
|
|
464
|
+
await self.recover_node(spec, source, credentials, connection, target, record)
|
|
465
|
+
points = await self.await_indexing(spec, connection.service, target)
|
|
466
|
+
except Exception as error:
|
|
467
|
+
return RestoredCollection(
|
|
468
|
+
name=target, original_name=original, status=RestorePhase.FAILED, error=str(error)
|
|
469
|
+
)
|
|
470
|
+
return RestoredCollection(
|
|
471
|
+
name=target,
|
|
472
|
+
original_name=original,
|
|
473
|
+
status=RestorePhase.COMPLETED,
|
|
474
|
+
size=format_size(backup.size_bytes),
|
|
475
|
+
points_count=points,
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
async def recover_node(
|
|
479
|
+
self,
|
|
480
|
+
spec: RestoreSpec,
|
|
481
|
+
source: RestoreSource,
|
|
482
|
+
credentials: S3Credentials,
|
|
483
|
+
connection: ClusterConnection,
|
|
484
|
+
target: str,
|
|
485
|
+
record: SnapshotRecord,
|
|
486
|
+
) -> None:
|
|
487
|
+
"""Hand the node a presigned URL so the snapshot flows bucket → node without touching us."""
|
|
488
|
+
location = await self.storage.presigned_get_url(
|
|
489
|
+
source.storage, credentials, record.key, PRESIGNED_URL_TTL_SECONDS
|
|
490
|
+
)
|
|
491
|
+
await self.qdrant.recover_snapshot(
|
|
492
|
+
connection.nodes[record.node_index], target, location, spec.priority, record.checksum
|
|
493
|
+
)
|
|
494
|
+
|
|
495
|
+
async def await_indexing(
|
|
496
|
+
self, spec: RestoreSpec, node: QdrantNode, collection: str
|
|
497
|
+
) -> int | None:
|
|
498
|
+
deadline = asyncio.get_running_loop().time() + self.indexing_timeout_seconds
|
|
499
|
+
while True:
|
|
500
|
+
info = await self.qdrant.collection_info(node, collection)
|
|
501
|
+
if not spec.wait_for_indexing or info.get("status") == "green":
|
|
502
|
+
return info.get("points_count")
|
|
503
|
+
if asyncio.get_running_loop().time() >= deadline:
|
|
504
|
+
raise TimeoutError(f"Collection {collection} did not turn green in time")
|
|
505
|
+
await asyncio.sleep(self.indexing_poll_seconds)
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
@dataclass
|
|
509
|
+
class ProcessSchedule:
|
|
510
|
+
kubernetes: KubernetesPort
|
|
511
|
+
|
|
512
|
+
async def execute(
|
|
513
|
+
self,
|
|
514
|
+
spec: BackupScheduleSpec,
|
|
515
|
+
status: BackupScheduleStatus,
|
|
516
|
+
owner: Mapping[str, Any],
|
|
517
|
+
now: datetime,
|
|
518
|
+
) -> BackupScheduleStatus:
|
|
519
|
+
records = await self.list_backups(spec)
|
|
520
|
+
await self.apply_retention(spec, records)
|
|
521
|
+
active = [r for r in records if not r.finished]
|
|
522
|
+
finished = [r for r in records if r.finished]
|
|
523
|
+
|
|
524
|
+
if spec.suspend:
|
|
525
|
+
return self.status_for(SchedulePhase.SUSPENDED, status, records, active, None)
|
|
526
|
+
|
|
527
|
+
baseline = status.last_schedule_time or parse_time(
|
|
528
|
+
owner["metadata"].get("creationTimestamp")
|
|
529
|
+
)
|
|
530
|
+
slot = spec.due(now, baseline)
|
|
531
|
+
launched = await self.launch(spec, owner, slot, active) if slot else None
|
|
532
|
+
last_schedule_time = slot if launched else status.last_schedule_time
|
|
533
|
+
next_time = spec.next_slot(now)
|
|
534
|
+
result = self.status_for(SchedulePhase.ACTIVE, status, records, active, next_time)
|
|
535
|
+
if not launched:
|
|
536
|
+
return replace(result, last_schedule_time=last_schedule_time)
|
|
537
|
+
return replace(
|
|
538
|
+
result,
|
|
539
|
+
last_schedule_time=last_schedule_time,
|
|
540
|
+
active_backup=launched,
|
|
541
|
+
last_backup_name=launched if not finished else result.last_backup_name,
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
async def list_backups(self, spec: BackupScheduleSpec) -> list[BackupRecord]:
|
|
545
|
+
bodies = await self.kubernetes.list_custom_resources(
|
|
546
|
+
BACKUPS, spec.namespace, spec.label_selector
|
|
547
|
+
)
|
|
548
|
+
return sorted(
|
|
549
|
+
(BackupRecord.from_resource(b) for b in bodies),
|
|
550
|
+
key=lambda r: r.creation_time,
|
|
551
|
+
reverse=True,
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
async def apply_retention(self, spec: BackupScheduleSpec, records: list[BackupRecord]) -> None:
|
|
555
|
+
for expired in spec.retention_policy.expired(r for r in records if r.finished):
|
|
556
|
+
await self.kubernetes.delete_custom_resource(
|
|
557
|
+
ResourceRef(BACKUPS, expired.name, spec.namespace)
|
|
558
|
+
)
|
|
559
|
+
records.remove(expired)
|
|
560
|
+
|
|
561
|
+
async def launch(
|
|
562
|
+
self,
|
|
563
|
+
spec: BackupScheduleSpec,
|
|
564
|
+
owner: Mapping[str, Any],
|
|
565
|
+
slot: datetime,
|
|
566
|
+
active: list[BackupRecord],
|
|
567
|
+
) -> str | None:
|
|
568
|
+
if active and spec.concurrency_policy == ConcurrencyPolicy.FORBID:
|
|
569
|
+
return None
|
|
570
|
+
if active and spec.concurrency_policy == ConcurrencyPolicy.REPLACE:
|
|
571
|
+
for record in active:
|
|
572
|
+
await self.kubernetes.delete_custom_resource(
|
|
573
|
+
ResourceRef(BACKUPS, record.name, spec.namespace)
|
|
574
|
+
)
|
|
575
|
+
name = spec.backup_name(slot)
|
|
576
|
+
await self.kubernetes.create_custom_resource(spec.to_backup_resource(name, owner))
|
|
577
|
+
return name
|
|
578
|
+
|
|
579
|
+
@staticmethod
|
|
580
|
+
def status_for(
|
|
581
|
+
phase: SchedulePhase,
|
|
582
|
+
status: BackupScheduleStatus,
|
|
583
|
+
records: list[BackupRecord],
|
|
584
|
+
active: list[BackupRecord],
|
|
585
|
+
next_time: datetime | None,
|
|
586
|
+
) -> BackupScheduleStatus:
|
|
587
|
+
latest = next((r for r in records if r.finished), None)
|
|
588
|
+
return BackupScheduleStatus(
|
|
589
|
+
phase=phase,
|
|
590
|
+
last_schedule_time=status.last_schedule_time,
|
|
591
|
+
last_backup_time=latest.completion_time if latest else None,
|
|
592
|
+
last_backup_name=latest.name if latest else None,
|
|
593
|
+
last_backup_status=latest.phase.value if latest else None,
|
|
594
|
+
next_backup_time=next_time,
|
|
595
|
+
active_backup=active[0].name if active else None,
|
|
596
|
+
recent_backups=tuple(records[:RECENT_BACKUPS_LIMIT]),
|
|
597
|
+
)
|