qdrant-operator 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qdrant_operator/__init__.py +1 -0
- qdrant_operator/container.py +49 -0
- qdrant_operator/domain.py +1245 -0
- qdrant_operator/handlers.py +189 -0
- qdrant_operator/helm_adapter.py +85 -0
- qdrant_operator/kubernetes_adapter.py +144 -0
- qdrant_operator/main.py +48 -0
- qdrant_operator/ports.py +116 -0
- qdrant_operator/py.typed +0 -0
- qdrant_operator/qdrant_adapter.py +124 -0
- qdrant_operator/s3_adapter.py +113 -0
- qdrant_operator/usecases.py +597 -0
- qdrant_operator-0.2.0.dist-info/METADATA +284 -0
- qdrant_operator-0.2.0.dist-info/RECORD +17 -0
- qdrant_operator-0.2.0.dist-info/WHEEL +4 -0
- qdrant_operator-0.2.0.dist-info/entry_points.txt +3 -0
- qdrant_operator-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1245 @@
|
|
|
1
|
+
"""Domain entities and value objects for the Qdrant operator."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from dataclasses import field
|
|
8
|
+
from dataclasses import replace
|
|
9
|
+
from datetime import UTC
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from datetime import timedelta
|
|
12
|
+
from enum import StrEnum
|
|
13
|
+
from typing import Any
|
|
14
|
+
from typing import cast
|
|
15
|
+
|
|
16
|
+
from croniter import croniter
|
|
17
|
+
|
|
18
|
+
GROUP = "qdrant.io"
|
|
19
|
+
VERSION = "v1alpha1"
|
|
20
|
+
API_VERSION = f"{GROUP}/{VERSION}"
|
|
21
|
+
SCHEDULE_LABEL = f"{GROUP}/schedule"
|
|
22
|
+
QDRANT_HTTP_PORT = 6333
|
|
23
|
+
CLUSTER_DOMAIN = "cluster.local"
|
|
24
|
+
MANIFEST_KEY = "manifest.json"
|
|
25
|
+
AUTO_GENERATED_API_KEY_KEY = "api-key"
|
|
26
|
+
TLS_MOUNT_PATH = "/qdrant/tls"
|
|
27
|
+
|
|
28
|
+
JsonDict = dict[str, Any]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ClusterPhase(StrEnum):
|
|
32
|
+
PENDING = "Pending"
|
|
33
|
+
RUNNING = "Running"
|
|
34
|
+
FAILED = "Failed"
|
|
35
|
+
UPGRADING = "Upgrading"
|
|
36
|
+
TERMINATING = "Terminating"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class BackupPhase(StrEnum):
|
|
40
|
+
PENDING = "Pending"
|
|
41
|
+
IN_PROGRESS = "InProgress"
|
|
42
|
+
COMPLETED = "Completed"
|
|
43
|
+
FAILED = "Failed"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class RestorePhase(StrEnum):
|
|
47
|
+
PENDING = "Pending"
|
|
48
|
+
DOWNLOADING = "Downloading"
|
|
49
|
+
RESTORING = "Restoring"
|
|
50
|
+
INDEXING = "Indexing"
|
|
51
|
+
COMPLETED = "Completed"
|
|
52
|
+
FAILED = "Failed"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class SchedulePhase(StrEnum):
|
|
56
|
+
ACTIVE = "Active"
|
|
57
|
+
SUSPENDED = "Suspended"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ConcurrencyPolicy(StrEnum):
|
|
61
|
+
ALLOW = "Allow"
|
|
62
|
+
FORBID = "Forbid"
|
|
63
|
+
REPLACE = "Replace"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class RestorePriority(StrEnum):
|
|
67
|
+
SNAPSHOT = "snapshot"
|
|
68
|
+
REPLICA = "replica"
|
|
69
|
+
NO_SYNC = "no_sync"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class ConditionStatus(StrEnum):
|
|
73
|
+
TRUE = "True"
|
|
74
|
+
FALSE = "False"
|
|
75
|
+
UNKNOWN = "Unknown"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class ResourceKind:
|
|
80
|
+
"""Identifies one of the operator's custom resource kinds."""
|
|
81
|
+
|
|
82
|
+
kind: str
|
|
83
|
+
plural: str
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def group(self) -> str:
|
|
87
|
+
return GROUP
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def version(self) -> str:
|
|
91
|
+
return VERSION
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
CLUSTERS = ResourceKind("QdrantCluster", "qdrantclusters")
|
|
95
|
+
BACKUPS = ResourceKind("QdrantBackup", "qdrantbackups")
|
|
96
|
+
SCHEDULES = ResourceKind("QdrantBackupSchedule", "qdrantbackupschedules")
|
|
97
|
+
RESTORES = ResourceKind("QdrantRestore", "qdrantrestores")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class ResourceRef:
|
|
102
|
+
"""Namespaced name of a custom resource."""
|
|
103
|
+
|
|
104
|
+
kind: ResourceKind
|
|
105
|
+
name: str
|
|
106
|
+
namespace: str
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def join_key(*parts: str) -> str:
|
|
110
|
+
"""Join S3 key parts, dropping empty segments and stray slashes."""
|
|
111
|
+
return "/".join(segment for part in parts for segment in part.split("/") if segment)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def format_size(size_bytes: int) -> str:
|
|
115
|
+
size = float(size_bytes)
|
|
116
|
+
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
117
|
+
if size < 1024:
|
|
118
|
+
return f"{size:.1f}{unit}"
|
|
119
|
+
size /= 1024
|
|
120
|
+
return f"{size:.1f}PB"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def parse_time(value: str | None) -> datetime | None:
|
|
124
|
+
if not value:
|
|
125
|
+
return None
|
|
126
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def format_time(value: datetime) -> str:
|
|
130
|
+
return value.astimezone(UTC).isoformat().replace("+00:00", "Z")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def release_name(cluster_name: str) -> str:
|
|
134
|
+
return f"qdrant-{cluster_name}"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def chart_version(version: str) -> str:
|
|
138
|
+
return version.removeprefix("v")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def drop_empty(values: Mapping[str, Any]) -> JsonDict:
|
|
142
|
+
"""Return a copy without None values, so serialized status dicts stay sparse."""
|
|
143
|
+
return {key: value for key, value in values.items() if value is not None}
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass(frozen=True)
|
|
147
|
+
class ResourceRequirements:
|
|
148
|
+
cpu: str | None = None
|
|
149
|
+
memory: str | None = None
|
|
150
|
+
|
|
151
|
+
@staticmethod
|
|
152
|
+
def from_dict(data: Mapping[str, Any]) -> "ResourceRequirements":
|
|
153
|
+
return ResourceRequirements(cpu=data.get("cpu"), memory=data.get("memory"))
|
|
154
|
+
|
|
155
|
+
def to_dict(self) -> JsonDict:
|
|
156
|
+
return drop_empty({"cpu": self.cpu, "memory": self.memory})
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
@dataclass(frozen=True)
|
|
160
|
+
class Resources:
|
|
161
|
+
requests: ResourceRequirements = field(default_factory=ResourceRequirements)
|
|
162
|
+
limits: ResourceRequirements = field(default_factory=ResourceRequirements)
|
|
163
|
+
|
|
164
|
+
@staticmethod
|
|
165
|
+
def from_dict(data: Mapping[str, Any]) -> "Resources":
|
|
166
|
+
return Resources(
|
|
167
|
+
requests=ResourceRequirements.from_dict(data.get("requests", {})),
|
|
168
|
+
limits=ResourceRequirements.from_dict(data.get("limits", {})),
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
def to_dict(self) -> JsonDict:
|
|
172
|
+
return {
|
|
173
|
+
key: value
|
|
174
|
+
for key, value in (
|
|
175
|
+
("requests", self.requests.to_dict()),
|
|
176
|
+
("limits", self.limits.to_dict()),
|
|
177
|
+
)
|
|
178
|
+
if value
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass(frozen=True)
|
|
183
|
+
class ImageSpec:
|
|
184
|
+
repository: str = "docker.io/qdrant/qdrant"
|
|
185
|
+
pull_policy: str = "IfNotPresent"
|
|
186
|
+
|
|
187
|
+
@staticmethod
|
|
188
|
+
def from_dict(data: Mapping[str, Any]) -> "ImageSpec":
|
|
189
|
+
return ImageSpec(
|
|
190
|
+
repository=data.get("repository", ImageSpec.repository),
|
|
191
|
+
pull_policy=data.get("pullPolicy", ImageSpec.pull_policy),
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass(frozen=True)
|
|
196
|
+
class PersistenceSpec:
|
|
197
|
+
size: str = "10Gi"
|
|
198
|
+
storage_class: str | None = None
|
|
199
|
+
access_modes: tuple[str, ...] = ("ReadWriteOnce",)
|
|
200
|
+
|
|
201
|
+
@staticmethod
|
|
202
|
+
def from_dict(data: Mapping[str, Any]) -> "PersistenceSpec":
|
|
203
|
+
return PersistenceSpec(
|
|
204
|
+
size=data.get("size", PersistenceSpec.size),
|
|
205
|
+
storage_class=data.get("storageClassName"),
|
|
206
|
+
access_modes=tuple(data.get("accessModes", PersistenceSpec.access_modes)),
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
def to_helm_values(self) -> JsonDict:
|
|
210
|
+
return drop_empty(
|
|
211
|
+
{
|
|
212
|
+
"size": self.size,
|
|
213
|
+
"accessModes": list(self.access_modes),
|
|
214
|
+
"storageClassName": self.storage_class,
|
|
215
|
+
}
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass(frozen=True)
|
|
220
|
+
class SnapshotPersistenceSpec:
|
|
221
|
+
enabled: bool = False
|
|
222
|
+
size: str = "10Gi"
|
|
223
|
+
storage_class: str | None = None
|
|
224
|
+
|
|
225
|
+
@staticmethod
|
|
226
|
+
def from_dict(data: Mapping[str, Any]) -> "SnapshotPersistenceSpec":
|
|
227
|
+
return SnapshotPersistenceSpec(
|
|
228
|
+
enabled=data.get("enabled", False),
|
|
229
|
+
size=data.get("size", SnapshotPersistenceSpec.size),
|
|
230
|
+
storage_class=data.get("storageClassName"),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def to_helm_values(self) -> JsonDict:
|
|
234
|
+
return drop_empty(
|
|
235
|
+
{"enabled": self.enabled, "size": self.size, "storageClassName": self.storage_class}
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@dataclass(frozen=True)
|
|
240
|
+
class SecretRef:
|
|
241
|
+
"""Reference to one key of a Kubernetes Secret."""
|
|
242
|
+
|
|
243
|
+
name: str
|
|
244
|
+
key: str
|
|
245
|
+
namespace: str
|
|
246
|
+
|
|
247
|
+
@staticmethod
|
|
248
|
+
def from_dict(data: Mapping[str, Any], namespace: str) -> "SecretRef":
|
|
249
|
+
return SecretRef(name=data["name"], key=data["key"], namespace=namespace)
|
|
250
|
+
|
|
251
|
+
def to_helm_value_from(self) -> JsonDict:
|
|
252
|
+
return {"valueFrom": {"secretKeyRef": {"name": self.name, "key": self.key}}}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
@dataclass(frozen=True)
|
|
256
|
+
class ApiKeySpec:
|
|
257
|
+
secret_ref: SecretRef | None = None
|
|
258
|
+
auto_generate: bool = False
|
|
259
|
+
|
|
260
|
+
@staticmethod
|
|
261
|
+
def from_dict(data: Mapping[str, Any], namespace: str) -> "ApiKeySpec":
|
|
262
|
+
secret_ref = data.get("secretRef")
|
|
263
|
+
return ApiKeySpec(
|
|
264
|
+
secret_ref=SecretRef.from_dict(secret_ref, namespace) if secret_ref else None,
|
|
265
|
+
auto_generate=data.get("autoGenerate", False),
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
def to_helm_value(self) -> JsonDict | bool:
|
|
269
|
+
if self.secret_ref:
|
|
270
|
+
return self.secret_ref.to_helm_value_from()
|
|
271
|
+
return self.auto_generate
|
|
272
|
+
|
|
273
|
+
def resolve_secret_ref(self, cluster_name: str, namespace: str) -> SecretRef | None:
|
|
274
|
+
"""Where the operator reads the key from: the user's secret or the chart-generated one."""
|
|
275
|
+
if self.secret_ref:
|
|
276
|
+
return self.secret_ref
|
|
277
|
+
if not self.auto_generate:
|
|
278
|
+
return None
|
|
279
|
+
return SecretRef(
|
|
280
|
+
name=f"{release_name(cluster_name)}-apikey",
|
|
281
|
+
key=AUTO_GENERATED_API_KEY_KEY,
|
|
282
|
+
namespace=namespace,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@dataclass(frozen=True)
|
|
287
|
+
class TlsSpec:
|
|
288
|
+
enabled: bool = False
|
|
289
|
+
secret_name: str | None = None
|
|
290
|
+
|
|
291
|
+
@staticmethod
|
|
292
|
+
def from_dict(data: Mapping[str, Any]) -> "TlsSpec":
|
|
293
|
+
return TlsSpec(
|
|
294
|
+
enabled=data.get("enabled", False),
|
|
295
|
+
secret_name=data.get("secretRef", {}).get("name"),
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
def ca_secret_ref(self, namespace: str) -> SecretRef | None:
|
|
299
|
+
if not self.enabled or not self.secret_name:
|
|
300
|
+
return None
|
|
301
|
+
return SecretRef(name=self.secret_name, key="ca.crt", namespace=namespace)
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
@dataclass(frozen=True)
|
|
305
|
+
class ServiceSpec:
|
|
306
|
+
type: str = "ClusterIP"
|
|
307
|
+
annotations: Mapping[str, str] = field(default_factory=dict[str, str])
|
|
308
|
+
|
|
309
|
+
@staticmethod
|
|
310
|
+
def from_dict(data: Mapping[str, Any]) -> "ServiceSpec":
|
|
311
|
+
return ServiceSpec(
|
|
312
|
+
type=data.get("type", ServiceSpec.type),
|
|
313
|
+
annotations=dict[str, str](data.get("annotations", {})),
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
def to_helm_values(self) -> JsonDict:
|
|
317
|
+
return {"type": self.type, "annotations": dict(self.annotations)}
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
@dataclass(frozen=True)
|
|
321
|
+
class ServiceMonitorSpec:
|
|
322
|
+
enabled: bool = False
|
|
323
|
+
interval: str = "30s"
|
|
324
|
+
labels: Mapping[str, str] = field(default_factory=dict[str, str])
|
|
325
|
+
|
|
326
|
+
@staticmethod
|
|
327
|
+
def from_dict(data: Mapping[str, Any]) -> "ServiceMonitorSpec":
|
|
328
|
+
return ServiceMonitorSpec(
|
|
329
|
+
enabled=data.get("enabled", False),
|
|
330
|
+
interval=data.get("interval", ServiceMonitorSpec.interval),
|
|
331
|
+
labels=dict[str, str](data.get("labels", {})),
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
@dataclass(frozen=True)
|
|
336
|
+
class MetricsSpec:
|
|
337
|
+
enabled: bool = False
|
|
338
|
+
service_monitor: ServiceMonitorSpec = field(default_factory=ServiceMonitorSpec)
|
|
339
|
+
|
|
340
|
+
@staticmethod
|
|
341
|
+
def from_dict(data: Mapping[str, Any]) -> "MetricsSpec":
|
|
342
|
+
return MetricsSpec(
|
|
343
|
+
enabled=data.get("enabled", False),
|
|
344
|
+
service_monitor=ServiceMonitorSpec.from_dict(data.get("serviceMonitor", {})),
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
def to_helm_values(self) -> JsonDict:
|
|
348
|
+
monitor_enabled = self.enabled and self.service_monitor.enabled
|
|
349
|
+
return {
|
|
350
|
+
"serviceMonitor": {
|
|
351
|
+
"enabled": monitor_enabled,
|
|
352
|
+
"scrapeInterval": self.service_monitor.interval,
|
|
353
|
+
"additionalLabels": dict(self.service_monitor.labels),
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
@dataclass(frozen=True)
|
|
359
|
+
class SchedulingSpec:
|
|
360
|
+
node_selector: Mapping[str, str] = field(default_factory=dict[str, str])
|
|
361
|
+
tolerations: tuple[JsonDict, ...] = ()
|
|
362
|
+
affinity: JsonDict = field(default_factory=dict[str, Any])
|
|
363
|
+
|
|
364
|
+
@staticmethod
|
|
365
|
+
def from_dict(spec: Mapping[str, Any]) -> "SchedulingSpec":
|
|
366
|
+
return SchedulingSpec(
|
|
367
|
+
node_selector=dict[str, str](spec.get("nodeSelector", {})),
|
|
368
|
+
tolerations=tuple[JsonDict, ...](spec.get("tolerations", [])),
|
|
369
|
+
affinity=dict[str, Any](spec.get("affinity", {})),
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
@dataclass(frozen=True)
|
|
374
|
+
class DistributedSpec:
|
|
375
|
+
enabled: bool = True
|
|
376
|
+
p2p_port: int = 6335
|
|
377
|
+
p2p_tls: bool = False
|
|
378
|
+
|
|
379
|
+
@staticmethod
|
|
380
|
+
def from_dict(data: Mapping[str, Any]) -> "DistributedSpec":
|
|
381
|
+
p2p = data.get("p2p", {})
|
|
382
|
+
return DistributedSpec(
|
|
383
|
+
enabled=data.get("enabled", True),
|
|
384
|
+
p2p_port=p2p.get("port", DistributedSpec.p2p_port),
|
|
385
|
+
p2p_tls=p2p.get("enableTls", False),
|
|
386
|
+
)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
@dataclass(frozen=True)
|
|
390
|
+
class ClusterSpec:
|
|
391
|
+
name: str
|
|
392
|
+
namespace: str
|
|
393
|
+
version: str
|
|
394
|
+
replicas: int = 1
|
|
395
|
+
image: ImageSpec = field(default_factory=ImageSpec)
|
|
396
|
+
resources: Resources = field(default_factory=Resources)
|
|
397
|
+
persistence: PersistenceSpec = field(default_factory=PersistenceSpec)
|
|
398
|
+
snapshot_persistence: SnapshotPersistenceSpec = field(default_factory=SnapshotPersistenceSpec)
|
|
399
|
+
distributed: DistributedSpec = field(default_factory=DistributedSpec)
|
|
400
|
+
service: ServiceSpec = field(default_factory=ServiceSpec)
|
|
401
|
+
api_key: ApiKeySpec = field(default_factory=ApiKeySpec)
|
|
402
|
+
read_only_api_key: ApiKeySpec = field(default_factory=ApiKeySpec)
|
|
403
|
+
tls: TlsSpec = field(default_factory=TlsSpec)
|
|
404
|
+
metrics: MetricsSpec = field(default_factory=MetricsSpec)
|
|
405
|
+
scheduling: SchedulingSpec = field(default_factory=SchedulingSpec)
|
|
406
|
+
config: JsonDict = field(default_factory=dict[str, Any])
|
|
407
|
+
|
|
408
|
+
@staticmethod
|
|
409
|
+
def from_dict(spec: Mapping[str, Any], meta: Mapping[str, Any]) -> "ClusterSpec":
|
|
410
|
+
namespace = meta["namespace"]
|
|
411
|
+
return ClusterSpec(
|
|
412
|
+
name=meta["name"],
|
|
413
|
+
namespace=namespace,
|
|
414
|
+
version=spec["version"],
|
|
415
|
+
replicas=spec.get("replicas", 1),
|
|
416
|
+
image=ImageSpec.from_dict(spec.get("image", {})),
|
|
417
|
+
resources=Resources.from_dict(spec.get("resources", {})),
|
|
418
|
+
persistence=PersistenceSpec.from_dict(spec.get("persistence", {})),
|
|
419
|
+
snapshot_persistence=SnapshotPersistenceSpec.from_dict(
|
|
420
|
+
spec.get("snapshotPersistence", {})
|
|
421
|
+
),
|
|
422
|
+
distributed=DistributedSpec.from_dict(spec.get("cluster", {})),
|
|
423
|
+
service=ServiceSpec.from_dict(spec.get("service", {})),
|
|
424
|
+
api_key=ApiKeySpec.from_dict(spec.get("apiKey", {}), namespace),
|
|
425
|
+
read_only_api_key=ApiKeySpec.from_dict(spec.get("readOnlyApiKey", {}), namespace),
|
|
426
|
+
tls=TlsSpec.from_dict(spec.get("tls", {})),
|
|
427
|
+
metrics=MetricsSpec.from_dict(spec.get("metrics", {})),
|
|
428
|
+
scheduling=SchedulingSpec.from_dict(spec),
|
|
429
|
+
config=dict[str, Any](spec.get("config", {})),
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
@property
|
|
433
|
+
def release_name(self) -> str:
|
|
434
|
+
return release_name(self.name)
|
|
435
|
+
|
|
436
|
+
@property
|
|
437
|
+
def scheme(self) -> str:
|
|
438
|
+
return "https" if self.tls.enabled else "http"
|
|
439
|
+
|
|
440
|
+
def service_url(self) -> str:
|
|
441
|
+
host = f"{self.release_name}.{self.namespace}.svc.{CLUSTER_DOMAIN}"
|
|
442
|
+
return f"{self.scheme}://{host}:{QDRANT_HTTP_PORT}"
|
|
443
|
+
|
|
444
|
+
def node_urls(self) -> tuple[str, ...]:
|
|
445
|
+
"""One URL per StatefulSet pod via the chart's headless service."""
|
|
446
|
+
headless = f"{self.release_name}-headless.{self.namespace}.svc.{CLUSTER_DOMAIN}"
|
|
447
|
+
return tuple(
|
|
448
|
+
f"{self.scheme}://{self.release_name}-{ordinal}.{headless}:{QDRANT_HTTP_PORT}"
|
|
449
|
+
for ordinal in range(self.replicas)
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
def to_helm_values(self) -> JsonDict:
|
|
453
|
+
config = merge_dicts(
|
|
454
|
+
{
|
|
455
|
+
"cluster": {
|
|
456
|
+
"enabled": self.distributed.enabled,
|
|
457
|
+
"p2p": {
|
|
458
|
+
"port": self.distributed.p2p_port,
|
|
459
|
+
"enable_tls": self.distributed.p2p_tls,
|
|
460
|
+
},
|
|
461
|
+
},
|
|
462
|
+
"service": {"enable_tls": self.tls.enabled},
|
|
463
|
+
},
|
|
464
|
+
self.tls_config(),
|
|
465
|
+
self.config,
|
|
466
|
+
)
|
|
467
|
+
values: JsonDict = {
|
|
468
|
+
"replicaCount": self.replicas,
|
|
469
|
+
"image": {"repository": self.image.repository, "pullPolicy": self.image.pull_policy},
|
|
470
|
+
"resources": self.resources.to_dict(),
|
|
471
|
+
"persistence": self.persistence.to_helm_values(),
|
|
472
|
+
"snapshotPersistence": self.snapshot_persistence.to_helm_values(),
|
|
473
|
+
"service": self.service.to_helm_values(),
|
|
474
|
+
"apiKey": self.api_key.to_helm_value(),
|
|
475
|
+
"readOnlyApiKey": self.read_only_api_key.to_helm_value(),
|
|
476
|
+
"metrics": self.metrics.to_helm_values(),
|
|
477
|
+
"nodeSelector": dict(self.scheduling.node_selector),
|
|
478
|
+
"tolerations": list(self.scheduling.tolerations),
|
|
479
|
+
"affinity": dict(self.scheduling.affinity),
|
|
480
|
+
"config": config,
|
|
481
|
+
}
|
|
482
|
+
return {**values, **self.tls_volumes()}
|
|
483
|
+
|
|
484
|
+
def tls_config(self) -> JsonDict:
|
|
485
|
+
if not self.tls.enabled:
|
|
486
|
+
return {}
|
|
487
|
+
return {"tls": {"cert": f"{TLS_MOUNT_PATH}/tls.crt", "key": f"{TLS_MOUNT_PATH}/tls.key"}}
|
|
488
|
+
|
|
489
|
+
def tls_volumes(self) -> JsonDict:
|
|
490
|
+
if not self.tls.enabled or not self.tls.secret_name:
|
|
491
|
+
return {}
|
|
492
|
+
return {
|
|
493
|
+
"additionalVolumes": [{"name": "tls", "secret": {"secretName": self.tls.secret_name}}],
|
|
494
|
+
"additionalVolumeMounts": [
|
|
495
|
+
{"name": "tls", "mountPath": TLS_MOUNT_PATH, "readOnly": True}
|
|
496
|
+
],
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def merge_dicts(*layers: Mapping[str, Any]) -> JsonDict:
|
|
501
|
+
"""Deep-merge mappings left to right; later layers win on scalar conflicts."""
|
|
502
|
+
merged: JsonDict = {}
|
|
503
|
+
for layer in layers:
|
|
504
|
+
for key, value in layer.items():
|
|
505
|
+
current = merged.get(key)
|
|
506
|
+
if isinstance(current, Mapping) and isinstance(value, Mapping):
|
|
507
|
+
merged[key] = merge_dicts(cast(JsonDict, current), cast(JsonDict, value))
|
|
508
|
+
continue
|
|
509
|
+
merged[key] = value
|
|
510
|
+
return merged
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
@dataclass(frozen=True)
|
|
514
|
+
class Condition:
|
|
515
|
+
type: str
|
|
516
|
+
status: ConditionStatus
|
|
517
|
+
last_transition_time: datetime
|
|
518
|
+
reason: str = ""
|
|
519
|
+
message: str = ""
|
|
520
|
+
|
|
521
|
+
@staticmethod
|
|
522
|
+
def from_dict(data: Mapping[str, Any]) -> "Condition":
|
|
523
|
+
return Condition(
|
|
524
|
+
type=data["type"],
|
|
525
|
+
status=ConditionStatus(data["status"]),
|
|
526
|
+
last_transition_time=parse_time(data.get("lastTransitionTime")) or datetime.now(UTC),
|
|
527
|
+
reason=data.get("reason", ""),
|
|
528
|
+
message=data.get("message", ""),
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
def to_dict(self) -> JsonDict:
|
|
532
|
+
return {
|
|
533
|
+
"type": self.type,
|
|
534
|
+
"status": self.status.value,
|
|
535
|
+
"lastTransitionTime": format_time(self.last_transition_time),
|
|
536
|
+
"reason": self.reason,
|
|
537
|
+
"message": self.message,
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def set_condition(conditions: Iterable[Condition], update: Condition) -> list[Condition]:
|
|
542
|
+
"""Replace the same-typed condition; keep its transition time when the status did not change."""
|
|
543
|
+
existing = {condition.type: condition for condition in conditions}
|
|
544
|
+
previous = existing.get(update.type)
|
|
545
|
+
unchanged = previous is not None and previous.status == update.status
|
|
546
|
+
merged = (
|
|
547
|
+
replace(update, last_transition_time=previous.last_transition_time)
|
|
548
|
+
if previous and unchanged
|
|
549
|
+
else update
|
|
550
|
+
)
|
|
551
|
+
return [*(c for c in existing.values() if c.type != update.type), merged]
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def conditions_from_dict(data: Mapping[str, Any]) -> list[Condition]:
|
|
555
|
+
return [Condition.from_dict(item) for item in data.get("conditions", [])]
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
@dataclass(frozen=True)
|
|
559
|
+
class ClusterStatus:
|
|
560
|
+
phase: ClusterPhase
|
|
561
|
+
replicas: int = 0
|
|
562
|
+
ready_replicas: int = 0
|
|
563
|
+
helm_release: str | None = None
|
|
564
|
+
endpoint: str | None = None
|
|
565
|
+
version: str | None = None
|
|
566
|
+
conditions: tuple[Condition, ...] = ()
|
|
567
|
+
observed_generation: int | None = None
|
|
568
|
+
|
|
569
|
+
@staticmethod
|
|
570
|
+
def from_dict(data: Mapping[str, Any]) -> "ClusterStatus":
|
|
571
|
+
return ClusterStatus(
|
|
572
|
+
phase=ClusterPhase(data.get("phase", ClusterPhase.PENDING)),
|
|
573
|
+
replicas=data.get("replicas", 0),
|
|
574
|
+
ready_replicas=data.get("readyReplicas", 0),
|
|
575
|
+
helm_release=data.get("helmRelease"),
|
|
576
|
+
endpoint=data.get("endpoint"),
|
|
577
|
+
version=data.get("version"),
|
|
578
|
+
conditions=tuple(conditions_from_dict(data)),
|
|
579
|
+
observed_generation=data.get("observedGeneration"),
|
|
580
|
+
)
|
|
581
|
+
|
|
582
|
+
def to_dict(self) -> JsonDict:
|
|
583
|
+
return {
|
|
584
|
+
"phase": self.phase.value,
|
|
585
|
+
"replicas": self.replicas,
|
|
586
|
+
"readyReplicas": self.ready_replicas,
|
|
587
|
+
"helmRelease": self.helm_release,
|
|
588
|
+
"endpoint": self.endpoint,
|
|
589
|
+
"version": self.version,
|
|
590
|
+
"conditions": [c.to_dict() for c in self.conditions],
|
|
591
|
+
"observedGeneration": self.observed_generation,
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
@dataclass(frozen=True)
|
|
596
|
+
class StatefulSetStatus:
|
|
597
|
+
replicas: int
|
|
598
|
+
ready_replicas: int
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
@dataclass(frozen=True)
|
|
602
|
+
class Snapshot:
|
|
603
|
+
name: str
|
|
604
|
+
collection: str
|
|
605
|
+
size_bytes: int
|
|
606
|
+
checksum: str | None = None
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
@dataclass(frozen=True)
|
|
610
|
+
class QdrantNode:
|
|
611
|
+
"""One reachable Qdrant HTTP endpoint together with what it takes to talk to it."""
|
|
612
|
+
|
|
613
|
+
url: str
|
|
614
|
+
api_key: str | None = None
|
|
615
|
+
ca_cert: str | None = None
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
@dataclass(frozen=True)
|
|
619
|
+
class ClusterConnection:
|
|
620
|
+
"""Resolved addresses of a QdrantCluster: the Service plus every StatefulSet pod."""
|
|
621
|
+
|
|
622
|
+
service: QdrantNode
|
|
623
|
+
nodes: tuple[QdrantNode, ...]
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
@dataclass(frozen=True)
|
|
627
|
+
class ClusterRef:
|
|
628
|
+
name: str
|
|
629
|
+
namespace: str
|
|
630
|
+
|
|
631
|
+
@staticmethod
|
|
632
|
+
def from_dict(data: Mapping[str, Any], default_namespace: str) -> "ClusterRef":
|
|
633
|
+
return ClusterRef(name=data["name"], namespace=data.get("namespace", default_namespace))
|
|
634
|
+
|
|
635
|
+
def to_dict(self) -> JsonDict:
|
|
636
|
+
return {"name": self.name, "namespace": self.namespace}
|
|
637
|
+
|
|
638
|
+
def to_resource_ref(self) -> ResourceRef:
|
|
639
|
+
return ResourceRef(CLUSTERS, self.name, self.namespace)
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
@dataclass(frozen=True)
|
|
643
|
+
class CredentialsSecretRef:
|
|
644
|
+
"""Secret holding both halves of an S3 credential pair."""
|
|
645
|
+
|
|
646
|
+
name: str
|
|
647
|
+
namespace: str
|
|
648
|
+
access_key_id_key: str = "AWS_ACCESS_KEY_ID"
|
|
649
|
+
secret_access_key_key: str = "AWS_SECRET_ACCESS_KEY"
|
|
650
|
+
|
|
651
|
+
@staticmethod
|
|
652
|
+
def from_dict(data: Mapping[str, Any], namespace: str) -> "CredentialsSecretRef":
|
|
653
|
+
return CredentialsSecretRef(
|
|
654
|
+
name=data["name"],
|
|
655
|
+
namespace=namespace,
|
|
656
|
+
access_key_id_key=data.get("accessKeyIdKey", CredentialsSecretRef.access_key_id_key),
|
|
657
|
+
secret_access_key_key=data.get(
|
|
658
|
+
"secretAccessKeyKey", CredentialsSecretRef.secret_access_key_key
|
|
659
|
+
),
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
def to_dict(self) -> JsonDict:
|
|
663
|
+
return {
|
|
664
|
+
"name": self.name,
|
|
665
|
+
"accessKeyIdKey": self.access_key_id_key,
|
|
666
|
+
"secretAccessKeyKey": self.secret_access_key_key,
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
def access_key_ref(self) -> SecretRef:
|
|
670
|
+
return SecretRef(self.name, self.access_key_id_key, self.namespace)
|
|
671
|
+
|
|
672
|
+
def secret_key_ref(self) -> SecretRef:
|
|
673
|
+
return SecretRef(self.name, self.secret_access_key_key, self.namespace)
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
@dataclass(frozen=True)
|
|
677
|
+
class S3Credentials:
|
|
678
|
+
access_key_id: str
|
|
679
|
+
secret_access_key: str
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
@dataclass(frozen=True)
|
|
683
|
+
class S3StorageSpec:
|
|
684
|
+
bucket: str
|
|
685
|
+
credentials_secret_ref: CredentialsSecretRef
|
|
686
|
+
prefix: str = ""
|
|
687
|
+
region: str = "us-east-1"
|
|
688
|
+
endpoint: str | None = None
|
|
689
|
+
force_path_style: bool = False
|
|
690
|
+
|
|
691
|
+
@staticmethod
|
|
692
|
+
def from_dict(data: Mapping[str, Any], namespace: str) -> "S3StorageSpec":
|
|
693
|
+
return S3StorageSpec(
|
|
694
|
+
bucket=data["bucket"],
|
|
695
|
+
credentials_secret_ref=CredentialsSecretRef.from_dict(
|
|
696
|
+
data["credentialsSecretRef"], namespace
|
|
697
|
+
),
|
|
698
|
+
prefix=data.get("path", data.get("prefix", "")),
|
|
699
|
+
region=data.get("region", S3StorageSpec.region),
|
|
700
|
+
endpoint=data.get("endpoint"),
|
|
701
|
+
force_path_style=data.get("forcePathStyle", False),
|
|
702
|
+
)
|
|
703
|
+
|
|
704
|
+
def to_dict(self) -> JsonDict:
|
|
705
|
+
return drop_empty(
|
|
706
|
+
{
|
|
707
|
+
"bucket": self.bucket,
|
|
708
|
+
"prefix": self.prefix,
|
|
709
|
+
"region": self.region,
|
|
710
|
+
"endpoint": self.endpoint,
|
|
711
|
+
"forcePathStyle": self.force_path_style,
|
|
712
|
+
"credentialsSecretRef": self.credentials_secret_ref.to_dict(),
|
|
713
|
+
}
|
|
714
|
+
)
|
|
715
|
+
|
|
716
|
+
def uri(self, *parts: str) -> str:
|
|
717
|
+
return f"s3://{join_key(self.bucket, self.prefix, *parts)}"
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
@dataclass(frozen=True)
|
|
721
|
+
class BackupSpec:
|
|
722
|
+
name: str
|
|
723
|
+
namespace: str
|
|
724
|
+
cluster_ref: ClusterRef
|
|
725
|
+
storage: S3StorageSpec
|
|
726
|
+
collections: tuple[str, ...] = ()
|
|
727
|
+
retention_days: int | None = None
|
|
728
|
+
|
|
729
|
+
@staticmethod
|
|
730
|
+
def from_dict(spec: Mapping[str, Any], meta: Mapping[str, Any]) -> "BackupSpec":
|
|
731
|
+
namespace = meta["namespace"]
|
|
732
|
+
return BackupSpec(
|
|
733
|
+
name=meta["name"],
|
|
734
|
+
namespace=namespace,
|
|
735
|
+
cluster_ref=ClusterRef.from_dict(spec["clusterRef"], namespace),
|
|
736
|
+
storage=S3StorageSpec.from_dict(spec["storage"]["s3"], namespace),
|
|
737
|
+
collections=tuple(spec.get("collections", [])),
|
|
738
|
+
retention_days=spec.get("retentionDays"),
|
|
739
|
+
)
|
|
740
|
+
|
|
741
|
+
@property
|
|
742
|
+
def root_key(self) -> str:
|
|
743
|
+
return join_key(self.storage.prefix, self.name)
|
|
744
|
+
|
|
745
|
+
def snapshot_key(self, collection: str, node_index: int, snapshot_name: str) -> str:
|
|
746
|
+
return join_key(self.root_key, collection, f"node-{node_index}", snapshot_name)
|
|
747
|
+
|
|
748
|
+
def expires_at(self, completion_time: datetime) -> datetime | None:
|
|
749
|
+
if not self.retention_days:
|
|
750
|
+
return None
|
|
751
|
+
return completion_time + timedelta(days=self.retention_days)
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
@dataclass(frozen=True)
|
|
755
|
+
class SnapshotRecord:
|
|
756
|
+
"""One stored snapshot: which node it came from and where it lives in the bucket."""
|
|
757
|
+
|
|
758
|
+
node_index: int
|
|
759
|
+
key: str
|
|
760
|
+
snapshot_name: str
|
|
761
|
+
size_bytes: int
|
|
762
|
+
checksum: str | None = None
|
|
763
|
+
|
|
764
|
+
@staticmethod
|
|
765
|
+
def from_dict(data: Mapping[str, Any]) -> "SnapshotRecord":
|
|
766
|
+
return SnapshotRecord(
|
|
767
|
+
node_index=data["nodeIndex"],
|
|
768
|
+
key=data["key"],
|
|
769
|
+
snapshot_name=data["snapshotName"],
|
|
770
|
+
size_bytes=data["sizeBytes"],
|
|
771
|
+
checksum=data.get("checksum"),
|
|
772
|
+
)
|
|
773
|
+
|
|
774
|
+
def to_dict(self) -> JsonDict:
|
|
775
|
+
return drop_empty(
|
|
776
|
+
{
|
|
777
|
+
"nodeIndex": self.node_index,
|
|
778
|
+
"key": self.key,
|
|
779
|
+
"snapshotName": self.snapshot_name,
|
|
780
|
+
"sizeBytes": self.size_bytes,
|
|
781
|
+
"checksum": self.checksum,
|
|
782
|
+
}
|
|
783
|
+
)
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
@dataclass(frozen=True)
|
|
787
|
+
class CollectionBackup:
|
|
788
|
+
name: str
|
|
789
|
+
snapshots: tuple[SnapshotRecord, ...]
|
|
790
|
+
|
|
791
|
+
@staticmethod
|
|
792
|
+
def from_dict(data: Mapping[str, Any]) -> "CollectionBackup":
|
|
793
|
+
return CollectionBackup(
|
|
794
|
+
name=data["name"],
|
|
795
|
+
snapshots=tuple(SnapshotRecord.from_dict(s) for s in data.get("snapshots", [])),
|
|
796
|
+
)
|
|
797
|
+
|
|
798
|
+
def to_dict(self) -> JsonDict:
|
|
799
|
+
return {"name": self.name, "snapshots": [s.to_dict() for s in self.snapshots]}
|
|
800
|
+
|
|
801
|
+
@property
|
|
802
|
+
def size_bytes(self) -> int:
|
|
803
|
+
return sum(s.size_bytes for s in self.snapshots)
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
@dataclass(frozen=True)
|
|
807
|
+
class BackupManifest:
|
|
808
|
+
"""Index of a completed backup, stored next to the snapshots so restores need no guessing."""
|
|
809
|
+
|
|
810
|
+
backup_name: str
|
|
811
|
+
cluster: ClusterRef
|
|
812
|
+
node_count: int
|
|
813
|
+
created_at: datetime
|
|
814
|
+
collections: tuple[CollectionBackup, ...]
|
|
815
|
+
|
|
816
|
+
@staticmethod
|
|
817
|
+
def from_dict(data: Mapping[str, Any]) -> "BackupManifest":
|
|
818
|
+
cluster = data["cluster"]
|
|
819
|
+
return BackupManifest(
|
|
820
|
+
backup_name=data["backupName"],
|
|
821
|
+
cluster=ClusterRef.from_dict(cluster, cluster["namespace"]),
|
|
822
|
+
node_count=data["nodeCount"],
|
|
823
|
+
created_at=parse_time(data["createdAt"]) or datetime.now(UTC),
|
|
824
|
+
collections=tuple(CollectionBackup.from_dict(c) for c in data.get("collections", [])),
|
|
825
|
+
)
|
|
826
|
+
|
|
827
|
+
def to_dict(self) -> JsonDict:
|
|
828
|
+
return {
|
|
829
|
+
"backupName": self.backup_name,
|
|
830
|
+
"cluster": self.cluster.to_dict(),
|
|
831
|
+
"nodeCount": self.node_count,
|
|
832
|
+
"createdAt": format_time(self.created_at),
|
|
833
|
+
"collections": [c.to_dict() for c in self.collections],
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
def collection(self, name: str) -> CollectionBackup | None:
|
|
837
|
+
return next((c for c in self.collections if c.name == name), None)
|
|
838
|
+
|
|
839
|
+
@property
|
|
840
|
+
def collection_names(self) -> tuple[str, ...]:
|
|
841
|
+
return tuple(c.name for c in self.collections)
|
|
842
|
+
|
|
843
|
+
@property
|
|
844
|
+
def size_bytes(self) -> int:
|
|
845
|
+
return sum(c.size_bytes for c in self.collections)
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
@dataclass(frozen=True)
|
|
849
|
+
class CollectionBackupStatus:
|
|
850
|
+
name: str
|
|
851
|
+
status: str
|
|
852
|
+
snapshots: tuple[SnapshotRecord, ...] = ()
|
|
853
|
+
error: str | None = None
|
|
854
|
+
|
|
855
|
+
def to_dict(self) -> JsonDict:
|
|
856
|
+
return {
|
|
857
|
+
"name": self.name,
|
|
858
|
+
"status": self.status,
|
|
859
|
+
"size": format_size(sum(s.size_bytes for s in self.snapshots)),
|
|
860
|
+
"snapshots": [
|
|
861
|
+
{"node": f"node-{s.node_index}", "snapshotName": s.snapshot_name, "key": s.key}
|
|
862
|
+
for s in self.snapshots
|
|
863
|
+
],
|
|
864
|
+
"error": self.error,
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
@dataclass(frozen=True)
|
|
869
|
+
class BackupStatus:
|
|
870
|
+
phase: BackupPhase
|
|
871
|
+
start_time: datetime | None = None
|
|
872
|
+
completion_time: datetime | None = None
|
|
873
|
+
expires_at: datetime | None = None
|
|
874
|
+
s3_path: str | None = None
|
|
875
|
+
total_size: str | None = None
|
|
876
|
+
collections: tuple[CollectionBackupStatus, ...] = ()
|
|
877
|
+
error: str | None = None
|
|
878
|
+
conditions: tuple[Condition, ...] = ()
|
|
879
|
+
|
|
880
|
+
@staticmethod
|
|
881
|
+
def from_dict(data: Mapping[str, Any]) -> "BackupStatus":
|
|
882
|
+
return BackupStatus(
|
|
883
|
+
phase=BackupPhase(data.get("phase", BackupPhase.PENDING)),
|
|
884
|
+
start_time=parse_time(data.get("startTime")),
|
|
885
|
+
completion_time=parse_time(data.get("completionTime")),
|
|
886
|
+
expires_at=parse_time(data.get("expiresAt")),
|
|
887
|
+
s3_path=data.get("s3Path"),
|
|
888
|
+
total_size=data.get("totalSize"),
|
|
889
|
+
error=data.get("error"),
|
|
890
|
+
conditions=tuple(conditions_from_dict(data)),
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
def to_dict(self) -> JsonDict:
|
|
894
|
+
return {
|
|
895
|
+
"phase": self.phase.value,
|
|
896
|
+
"startTime": format_time(self.start_time) if self.start_time else None,
|
|
897
|
+
"completionTime": (format_time(self.completion_time) if self.completion_time else None),
|
|
898
|
+
"expiresAt": format_time(self.expires_at) if self.expires_at else None,
|
|
899
|
+
"s3Path": self.s3_path,
|
|
900
|
+
"totalSize": self.total_size,
|
|
901
|
+
"collections": [c.to_dict() for c in self.collections],
|
|
902
|
+
"error": self.error,
|
|
903
|
+
"conditions": [c.to_dict() for c in self.conditions],
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
@property
|
|
907
|
+
def finished(self) -> bool:
|
|
908
|
+
return self.phase in (BackupPhase.COMPLETED, BackupPhase.FAILED)
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
@dataclass(frozen=True)
|
|
912
|
+
class RetentionPolicy:
|
|
913
|
+
keep_last: int | None = None
|
|
914
|
+
keep_daily: int | None = None
|
|
915
|
+
keep_weekly: int | None = None
|
|
916
|
+
keep_monthly: int | None = None
|
|
917
|
+
|
|
918
|
+
@staticmethod
|
|
919
|
+
def from_dict(data: Mapping[str, Any]) -> "RetentionPolicy":
|
|
920
|
+
return RetentionPolicy(
|
|
921
|
+
keep_last=data.get("keepLast"),
|
|
922
|
+
keep_daily=data.get("keepDaily"),
|
|
923
|
+
keep_weekly=data.get("keepWeekly"),
|
|
924
|
+
keep_monthly=data.get("keepMonthly"),
|
|
925
|
+
)
|
|
926
|
+
|
|
927
|
+
@property
|
|
928
|
+
def configured(self) -> bool:
|
|
929
|
+
return any(
|
|
930
|
+
value is not None
|
|
931
|
+
for value in (self.keep_last, self.keep_daily, self.keep_weekly, self.keep_monthly)
|
|
932
|
+
)
|
|
933
|
+
|
|
934
|
+
def expired(self, backups: Iterable["BackupRecord"]) -> list["BackupRecord"]:
|
|
935
|
+
"""Backups no bucket of the policy keeps, newest-first grandfather-father-son selection."""
|
|
936
|
+
if not self.configured:
|
|
937
|
+
return []
|
|
938
|
+
ordered = sorted(backups, key=lambda b: b.creation_time, reverse=True)
|
|
939
|
+
keep = {b.name for b in ordered[: self.keep_last or 0]}
|
|
940
|
+
keep |= newest_per_bucket(ordered, self.keep_daily, lambda t: t.strftime("%Y-%m-%d"))
|
|
941
|
+
keep |= newest_per_bucket(ordered, self.keep_weekly, lambda t: t.strftime("%G-W%V"))
|
|
942
|
+
keep |= newest_per_bucket(ordered, self.keep_monthly, lambda t: t.strftime("%Y-%m"))
|
|
943
|
+
return [b for b in ordered if b.name not in keep]
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def newest_per_bucket(
|
|
947
|
+
ordered: list["BackupRecord"],
|
|
948
|
+
count: int | None,
|
|
949
|
+
bucket_of: Callable[[datetime], str],
|
|
950
|
+
) -> set[str]:
|
|
951
|
+
if not count:
|
|
952
|
+
return set()
|
|
953
|
+
winners: dict[str, str] = {}
|
|
954
|
+
for backup in ordered:
|
|
955
|
+
winners.setdefault(bucket_of(backup.creation_time), backup.name)
|
|
956
|
+
return set(list(winners.values())[:count])
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
@dataclass(frozen=True)
|
|
960
|
+
class BackupRecord:
|
|
961
|
+
"""What a schedule needs to know about one of its QdrantBackups."""
|
|
962
|
+
|
|
963
|
+
name: str
|
|
964
|
+
creation_time: datetime
|
|
965
|
+
phase: BackupPhase
|
|
966
|
+
completion_time: datetime | None = None
|
|
967
|
+
total_size: str | None = None
|
|
968
|
+
|
|
969
|
+
@staticmethod
|
|
970
|
+
def from_resource(body: Mapping[str, Any]) -> "BackupRecord":
|
|
971
|
+
status = BackupStatus.from_dict(body.get("status", {}))
|
|
972
|
+
return BackupRecord(
|
|
973
|
+
name=body["metadata"]["name"],
|
|
974
|
+
creation_time=parse_time(body["metadata"].get("creationTimestamp"))
|
|
975
|
+
or datetime.now(UTC),
|
|
976
|
+
phase=status.phase,
|
|
977
|
+
completion_time=status.completion_time,
|
|
978
|
+
total_size=status.total_size,
|
|
979
|
+
)
|
|
980
|
+
|
|
981
|
+
@property
|
|
982
|
+
def finished(self) -> bool:
|
|
983
|
+
return self.phase in (BackupPhase.COMPLETED, BackupPhase.FAILED)
|
|
984
|
+
|
|
985
|
+
def to_dict(self) -> JsonDict:
|
|
986
|
+
return {
|
|
987
|
+
"name": self.name,
|
|
988
|
+
"creationTime": format_time(self.creation_time),
|
|
989
|
+
"completionTime": (format_time(self.completion_time) if self.completion_time else None),
|
|
990
|
+
"status": self.phase.value,
|
|
991
|
+
"size": self.total_size,
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
@dataclass(frozen=True)
|
|
996
|
+
class BackupScheduleSpec:
|
|
997
|
+
name: str
|
|
998
|
+
namespace: str
|
|
999
|
+
schedule: str
|
|
1000
|
+
cluster_ref: ClusterRef
|
|
1001
|
+
storage: S3StorageSpec
|
|
1002
|
+
collections: tuple[str, ...] = ()
|
|
1003
|
+
retention_policy: RetentionPolicy = field(default_factory=RetentionPolicy)
|
|
1004
|
+
suspend: bool = False
|
|
1005
|
+
concurrency_policy: ConcurrencyPolicy = ConcurrencyPolicy.FORBID
|
|
1006
|
+
starting_deadline_seconds: int | None = None
|
|
1007
|
+
|
|
1008
|
+
@staticmethod
|
|
1009
|
+
def from_dict(spec: Mapping[str, Any], meta: Mapping[str, Any]) -> "BackupScheduleSpec":
|
|
1010
|
+
namespace = meta["namespace"]
|
|
1011
|
+
return BackupScheduleSpec(
|
|
1012
|
+
name=meta["name"],
|
|
1013
|
+
namespace=namespace,
|
|
1014
|
+
schedule=spec["schedule"],
|
|
1015
|
+
cluster_ref=ClusterRef.from_dict(spec["clusterRef"], namespace),
|
|
1016
|
+
storage=S3StorageSpec.from_dict(spec["storage"]["s3"], namespace),
|
|
1017
|
+
collections=tuple(spec.get("collections", [])),
|
|
1018
|
+
retention_policy=RetentionPolicy.from_dict(spec.get("retentionPolicy", {})),
|
|
1019
|
+
suspend=spec.get("suspend", False),
|
|
1020
|
+
concurrency_policy=ConcurrencyPolicy(spec.get("concurrencyPolicy", "Forbid")),
|
|
1021
|
+
starting_deadline_seconds=spec.get("startingDeadlineSeconds"),
|
|
1022
|
+
)
|
|
1023
|
+
|
|
1024
|
+
@property
|
|
1025
|
+
def label_selector(self) -> str:
|
|
1026
|
+
return f"{SCHEDULE_LABEL}={self.name}"
|
|
1027
|
+
|
|
1028
|
+
def backup_name(self, at: datetime) -> str:
|
|
1029
|
+
return f"{self.name}-{at.strftime('%Y%m%d-%H%M%S')}"
|
|
1030
|
+
|
|
1031
|
+
def previous_slot(self, now: datetime) -> datetime:
|
|
1032
|
+
return croniter(self.schedule, now).get_prev(datetime)
|
|
1033
|
+
|
|
1034
|
+
def next_slot(self, now: datetime) -> datetime:
|
|
1035
|
+
return croniter(self.schedule, now).get_next(datetime)
|
|
1036
|
+
|
|
1037
|
+
def due(self, now: datetime, baseline: datetime | None) -> datetime | None:
|
|
1038
|
+
"""The missed slot after `baseline` (last run or creation), or None when up to date."""
|
|
1039
|
+
slot = self.previous_slot(now)
|
|
1040
|
+
if baseline and baseline >= slot:
|
|
1041
|
+
return None
|
|
1042
|
+
if self.starting_deadline_seconds is None:
|
|
1043
|
+
return slot
|
|
1044
|
+
if now - slot > timedelta(seconds=self.starting_deadline_seconds):
|
|
1045
|
+
return None
|
|
1046
|
+
return slot
|
|
1047
|
+
|
|
1048
|
+
def to_backup_resource(self, backup_name: str, owner: Mapping[str, Any]) -> JsonDict:
|
|
1049
|
+
"""Body of the QdrantBackup this schedule spawns; owned by the schedule for GC."""
|
|
1050
|
+
spec: JsonDict = {
|
|
1051
|
+
"clusterRef": self.cluster_ref.to_dict(),
|
|
1052
|
+
"storage": {"s3": self.storage.to_dict()},
|
|
1053
|
+
}
|
|
1054
|
+
if self.collections:
|
|
1055
|
+
spec["collections"] = list(self.collections)
|
|
1056
|
+
return {
|
|
1057
|
+
"apiVersion": API_VERSION,
|
|
1058
|
+
"kind": BACKUPS.kind,
|
|
1059
|
+
"metadata": {
|
|
1060
|
+
"name": backup_name,
|
|
1061
|
+
"namespace": self.namespace,
|
|
1062
|
+
"labels": {SCHEDULE_LABEL: self.name},
|
|
1063
|
+
"ownerReferences": [owner_reference(owner)],
|
|
1064
|
+
},
|
|
1065
|
+
"spec": spec,
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def owner_reference(owner: Mapping[str, Any]) -> JsonDict:
|
|
1070
|
+
metadata = owner["metadata"]
|
|
1071
|
+
return {
|
|
1072
|
+
"apiVersion": owner["apiVersion"],
|
|
1073
|
+
"kind": owner["kind"],
|
|
1074
|
+
"name": metadata["name"],
|
|
1075
|
+
"uid": metadata["uid"],
|
|
1076
|
+
"controller": True,
|
|
1077
|
+
"blockOwnerDeletion": True,
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
@dataclass(frozen=True)
|
|
1082
|
+
class BackupScheduleStatus:
|
|
1083
|
+
phase: SchedulePhase
|
|
1084
|
+
last_schedule_time: datetime | None = None
|
|
1085
|
+
last_backup_time: datetime | None = None
|
|
1086
|
+
last_backup_name: str | None = None
|
|
1087
|
+
last_backup_status: str | None = None
|
|
1088
|
+
next_backup_time: datetime | None = None
|
|
1089
|
+
active_backup: str | None = None
|
|
1090
|
+
recent_backups: tuple[BackupRecord, ...] = ()
|
|
1091
|
+
conditions: tuple[Condition, ...] = ()
|
|
1092
|
+
|
|
1093
|
+
@staticmethod
|
|
1094
|
+
def from_dict(data: Mapping[str, Any]) -> "BackupScheduleStatus":
|
|
1095
|
+
return BackupScheduleStatus(
|
|
1096
|
+
phase=SchedulePhase(data.get("phase", SchedulePhase.ACTIVE)),
|
|
1097
|
+
last_schedule_time=parse_time(data.get("lastScheduleTime")),
|
|
1098
|
+
last_backup_time=parse_time(data.get("lastBackupTime")),
|
|
1099
|
+
last_backup_name=data.get("lastBackupName"),
|
|
1100
|
+
last_backup_status=data.get("lastBackupStatus"),
|
|
1101
|
+
next_backup_time=parse_time(data.get("nextBackupTime")),
|
|
1102
|
+
active_backup=data.get("activeBackup"),
|
|
1103
|
+
conditions=tuple(conditions_from_dict(data)),
|
|
1104
|
+
)
|
|
1105
|
+
|
|
1106
|
+
def to_dict(self) -> JsonDict:
|
|
1107
|
+
return {
|
|
1108
|
+
"phase": self.phase.value,
|
|
1109
|
+
"lastScheduleTime": (
|
|
1110
|
+
format_time(self.last_schedule_time) if self.last_schedule_time else None
|
|
1111
|
+
),
|
|
1112
|
+
"lastBackupTime": (
|
|
1113
|
+
format_time(self.last_backup_time) if self.last_backup_time else None
|
|
1114
|
+
),
|
|
1115
|
+
"lastBackupName": self.last_backup_name,
|
|
1116
|
+
"lastBackupStatus": self.last_backup_status,
|
|
1117
|
+
"nextBackupTime": (
|
|
1118
|
+
format_time(self.next_backup_time) if self.next_backup_time else None
|
|
1119
|
+
),
|
|
1120
|
+
"activeBackup": self.active_backup,
|
|
1121
|
+
"recentBackups": [b.to_dict() for b in self.recent_backups],
|
|
1122
|
+
"conditions": [c.to_dict() for c in self.conditions],
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
|
|
1126
|
+
@dataclass(frozen=True)
|
|
1127
|
+
class BackupRef:
|
|
1128
|
+
name: str
|
|
1129
|
+
namespace: str
|
|
1130
|
+
|
|
1131
|
+
@staticmethod
|
|
1132
|
+
def from_dict(data: Mapping[str, Any], default_namespace: str) -> "BackupRef":
|
|
1133
|
+
return BackupRef(name=data["name"], namespace=data.get("namespace", default_namespace))
|
|
1134
|
+
|
|
1135
|
+
def to_resource_ref(self) -> ResourceRef:
|
|
1136
|
+
return ResourceRef(BACKUPS, self.name, self.namespace)
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
@dataclass(frozen=True)
|
|
1140
|
+
class RestoreSpec:
|
|
1141
|
+
name: str
|
|
1142
|
+
namespace: str
|
|
1143
|
+
target_cluster_ref: ClusterRef
|
|
1144
|
+
backup_ref: BackupRef | None = None
|
|
1145
|
+
source_s3: S3StorageSpec | None = None
|
|
1146
|
+
collections: tuple[str, ...] = ()
|
|
1147
|
+
collection_mapping: Mapping[str, str] = field(default_factory=dict[str, str])
|
|
1148
|
+
priority: RestorePriority = RestorePriority.SNAPSHOT
|
|
1149
|
+
wait_for_indexing: bool = True
|
|
1150
|
+
|
|
1151
|
+
@staticmethod
|
|
1152
|
+
def from_dict(spec: Mapping[str, Any], meta: Mapping[str, Any]) -> "RestoreSpec":
|
|
1153
|
+
namespace = meta["namespace"]
|
|
1154
|
+
backup_ref = spec.get("backupRef")
|
|
1155
|
+
source_s3 = spec.get("source", {}).get("s3")
|
|
1156
|
+
if not backup_ref and not source_s3:
|
|
1157
|
+
raise ValueError("QdrantRestore needs either spec.backupRef or spec.source.s3")
|
|
1158
|
+
return RestoreSpec(
|
|
1159
|
+
name=meta["name"],
|
|
1160
|
+
namespace=namespace,
|
|
1161
|
+
target_cluster_ref=ClusterRef.from_dict(spec["targetClusterRef"], namespace),
|
|
1162
|
+
backup_ref=BackupRef.from_dict(backup_ref, namespace) if backup_ref else None,
|
|
1163
|
+
source_s3=S3StorageSpec.from_dict(source_s3, namespace) if source_s3 else None,
|
|
1164
|
+
collections=tuple(spec.get("collections", [])),
|
|
1165
|
+
collection_mapping=dict[str, str](spec.get("collectionMapping", {})),
|
|
1166
|
+
priority=RestorePriority(spec.get("priority", RestorePriority.SNAPSHOT)),
|
|
1167
|
+
wait_for_indexing=spec.get("waitForIndexing", True),
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
def target_name(self, collection: str) -> str:
|
|
1171
|
+
return self.collection_mapping.get(collection, collection)
|
|
1172
|
+
|
|
1173
|
+
def select_collections(self, available: Iterable[str]) -> tuple[str, ...]:
|
|
1174
|
+
names = tuple(available)
|
|
1175
|
+
if not self.collections:
|
|
1176
|
+
return names
|
|
1177
|
+
missing = [c for c in self.collections if c not in names]
|
|
1178
|
+
if missing:
|
|
1179
|
+
raise ValueError(f"Collections not present in backup: {', '.join(missing)}")
|
|
1180
|
+
return self.collections
|
|
1181
|
+
|
|
1182
|
+
|
|
1183
|
+
@dataclass(frozen=True)
|
|
1184
|
+
class RestoredCollection:
|
|
1185
|
+
name: str
|
|
1186
|
+
status: str
|
|
1187
|
+
original_name: str | None = None
|
|
1188
|
+
size: str | None = None
|
|
1189
|
+
points_count: int | None = None
|
|
1190
|
+
error: str | None = None
|
|
1191
|
+
|
|
1192
|
+
def to_dict(self) -> JsonDict:
|
|
1193
|
+
return {
|
|
1194
|
+
"name": self.name,
|
|
1195
|
+
"originalName": self.original_name,
|
|
1196
|
+
"status": self.status,
|
|
1197
|
+
"size": self.size,
|
|
1198
|
+
"pointsCount": self.points_count,
|
|
1199
|
+
"error": self.error,
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
|
|
1203
|
+
@dataclass(frozen=True)
|
|
1204
|
+
class RestoreProgress:
|
|
1205
|
+
collections_total: int = 0
|
|
1206
|
+
collections_completed: int = 0
|
|
1207
|
+
current_collection: str | None = None
|
|
1208
|
+
|
|
1209
|
+
@property
|
|
1210
|
+
def percentage(self) -> int:
|
|
1211
|
+
if not self.collections_total:
|
|
1212
|
+
return 0
|
|
1213
|
+
return round(100 * self.collections_completed / self.collections_total)
|
|
1214
|
+
|
|
1215
|
+
def to_dict(self) -> JsonDict:
|
|
1216
|
+
return {
|
|
1217
|
+
"collectionsTotal": self.collections_total,
|
|
1218
|
+
"collectionsCompleted": self.collections_completed,
|
|
1219
|
+
"currentCollection": self.current_collection,
|
|
1220
|
+
"percentage": self.percentage,
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
@dataclass(frozen=True)
|
|
1225
|
+
class RestoreStatus:
|
|
1226
|
+
phase: RestorePhase
|
|
1227
|
+
start_time: datetime | None = None
|
|
1228
|
+
completion_time: datetime | None = None
|
|
1229
|
+
source_backup: str | None = None
|
|
1230
|
+
restored_collections: tuple[RestoredCollection, ...] = ()
|
|
1231
|
+
progress: RestoreProgress = field(default_factory=RestoreProgress)
|
|
1232
|
+
error: str | None = None
|
|
1233
|
+
conditions: tuple[Condition, ...] = ()
|
|
1234
|
+
|
|
1235
|
+
def to_dict(self) -> JsonDict:
|
|
1236
|
+
return {
|
|
1237
|
+
"phase": self.phase.value,
|
|
1238
|
+
"startTime": format_time(self.start_time) if self.start_time else None,
|
|
1239
|
+
"completionTime": (format_time(self.completion_time) if self.completion_time else None),
|
|
1240
|
+
"sourceBackup": self.source_backup,
|
|
1241
|
+
"restoredCollections": [c.to_dict() for c in self.restored_collections],
|
|
1242
|
+
"progress": self.progress.to_dict(),
|
|
1243
|
+
"error": self.error,
|
|
1244
|
+
"conditions": [c.to_dict() for c in self.conditions],
|
|
1245
|
+
}
|