cnpg-drill 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cnpg_drill/__init__.py +3 -0
- cnpg_drill/cli.py +43 -0
- cnpg_drill/core.py +393 -0
- cnpg_drill-0.1.2.dist-info/METADATA +131 -0
- cnpg_drill-0.1.2.dist-info/RECORD +9 -0
- cnpg_drill-0.1.2.dist-info/WHEEL +5 -0
- cnpg_drill-0.1.2.dist-info/entry_points.txt +2 -0
- cnpg_drill-0.1.2.dist-info/licenses/LICENSE +21 -0
- cnpg_drill-0.1.2.dist-info/top_level.txt +1 -0
cnpg_drill/__init__.py
ADDED
cnpg_drill/cli.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Command-line interface for local use and Kubernetes CronJobs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
from .core import Config, DrillError, Kubectl, prepare, run_drill
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main(argv: list[str] | None = None) -> int:
|
|
15
|
+
parser = argparse.ArgumentParser(prog="cnpg-drill", description="Verify a CloudNativePG backup with a disposable recovery cluster")
|
|
16
|
+
parser.add_argument("--version", action="version", version=__version__)
|
|
17
|
+
parser.add_argument("--kubectl", default="kubectl", help="kubectl executable path")
|
|
18
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
19
|
+
for command in ("plan", "run"):
|
|
20
|
+
item = sub.add_parser(command)
|
|
21
|
+
item.add_argument("--config", required=True, type=Path, help="JSON drill config")
|
|
22
|
+
if command == "run":
|
|
23
|
+
item.add_argument("--report", type=Path, help="write JSON report to this path")
|
|
24
|
+
args = parser.parse_args(argv)
|
|
25
|
+
try:
|
|
26
|
+
config = Config.from_dict(json.loads(args.config.read_text(encoding="utf-8")))
|
|
27
|
+
client = Kubectl(args.kubectl)
|
|
28
|
+
if args.command == "plan":
|
|
29
|
+
print(json.dumps(prepare(client, config), indent=2, sort_keys=True))
|
|
30
|
+
return 0
|
|
31
|
+
report = run_drill(client, config)
|
|
32
|
+
encoded = json.dumps(report, indent=2, sort_keys=True)
|
|
33
|
+
print(encoded)
|
|
34
|
+
if args.report:
|
|
35
|
+
args.report.write_text(encoded + "\n", encoding="utf-8")
|
|
36
|
+
return 0 if report["status"] == "passed" else (2 if report.get("phase") == "preflight" else 1)
|
|
37
|
+
except (DrillError, OSError, json.JSONDecodeError) as exc:
|
|
38
|
+
print(f"cnpg-drill: {exc}", file=sys.stderr)
|
|
39
|
+
return 2
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
if __name__ == "__main__":
|
|
43
|
+
raise SystemExit(main())
|
cnpg_drill/core.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""Recovery planning and execution, with a deliberately small Kubernetes surface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import datetime as dt
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
import secrets
|
|
11
|
+
import subprocess
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Any, Callable
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
PLUGIN = "barman-cloud.cloudnative-pg.io"
|
|
18
|
+
API = "clusters.postgresql.cnpg.io"
|
|
19
|
+
DNS_NAME = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DrillError(RuntimeError):
|
|
23
|
+
"""A recoverable validation or drill failure."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class Check:
|
|
28
|
+
name: str
|
|
29
|
+
query: str
|
|
30
|
+
expected: str | None = None
|
|
31
|
+
max_age_seconds: int | None = None
|
|
32
|
+
database: str = "postgres"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class Config:
|
|
37
|
+
namespace: str
|
|
38
|
+
cluster: str
|
|
39
|
+
recovery_object_store: str | None = None
|
|
40
|
+
timeout_seconds: int = 1800
|
|
41
|
+
poll_seconds: int = 5
|
|
42
|
+
max_backup_age_seconds: int = 691200
|
|
43
|
+
target_time: str | None = None
|
|
44
|
+
checks: tuple[Check, ...] = (Check("connection", "SELECT 1", "1"),)
|
|
45
|
+
retain_on_failure: bool = False
|
|
46
|
+
|
|
47
|
+
@classmethod
|
|
48
|
+
def from_dict(cls, raw: dict[str, Any]) -> Config:
|
|
49
|
+
if not isinstance(raw, dict):
|
|
50
|
+
raise DrillError("Config must be a JSON object")
|
|
51
|
+
allowed = {"namespace", "cluster", "recoveryObjectStore", "timeoutSeconds", "pollSeconds", "maxBackupAgeSeconds", "targetTime", "checks", "retainOnFailure"}
|
|
52
|
+
unknown = set(raw) - allowed
|
|
53
|
+
if unknown:
|
|
54
|
+
raise DrillError(f"Unknown config keys: {', '.join(sorted(unknown))}")
|
|
55
|
+
namespace, cluster = raw.get("namespace"), raw.get("cluster")
|
|
56
|
+
for label, value in (("namespace", namespace), ("cluster", cluster)):
|
|
57
|
+
if not isinstance(value, str) or not DNS_NAME.fullmatch(value) or len(value) > 63:
|
|
58
|
+
raise DrillError(f"{label} must be a Kubernetes DNS label")
|
|
59
|
+
recovery_object_store = raw.get("recoveryObjectStore")
|
|
60
|
+
if recovery_object_store is not None and (not isinstance(recovery_object_store, str) or not DNS_NAME.fullmatch(recovery_object_store) or len(recovery_object_store) > 63):
|
|
61
|
+
raise DrillError("recoveryObjectStore must be a Kubernetes DNS label")
|
|
62
|
+
timeout = raw.get("timeoutSeconds", 1800)
|
|
63
|
+
poll = raw.get("pollSeconds", 5)
|
|
64
|
+
max_backup_age = raw.get("maxBackupAgeSeconds", 691200)
|
|
65
|
+
if type(timeout) is not int or not 30 <= timeout <= 86400:
|
|
66
|
+
raise DrillError("timeoutSeconds must be 30..86400")
|
|
67
|
+
if type(poll) is not int or not 1 <= poll <= 60:
|
|
68
|
+
raise DrillError("pollSeconds must be 1..60")
|
|
69
|
+
if type(max_backup_age) is not int or not 1 <= max_backup_age <= 31536000:
|
|
70
|
+
raise DrillError("maxBackupAgeSeconds must be 1..31536000")
|
|
71
|
+
target = raw.get("targetTime")
|
|
72
|
+
if target is not None:
|
|
73
|
+
if not isinstance(target, str):
|
|
74
|
+
raise DrillError("targetTime must be an RFC3339 timestamp")
|
|
75
|
+
try:
|
|
76
|
+
parsed = dt.datetime.fromisoformat(target.replace("Z", "+00:00"))
|
|
77
|
+
except ValueError as exc:
|
|
78
|
+
raise DrillError("targetTime must be an RFC3339 timestamp") from exc
|
|
79
|
+
if parsed.tzinfo is None:
|
|
80
|
+
raise DrillError("targetTime must include a timezone")
|
|
81
|
+
if parsed > dt.datetime.now(dt.timezone.utc):
|
|
82
|
+
raise DrillError("targetTime cannot be in the future")
|
|
83
|
+
checks_raw = raw.get("checks", [{"name": "connection", "query": "SELECT 1", "expected": "1"}])
|
|
84
|
+
if not isinstance(checks_raw, list) or not 1 <= len(checks_raw) <= 50:
|
|
85
|
+
raise DrillError("checks must be a list of 1..50 checks")
|
|
86
|
+
checks: list[Check] = []
|
|
87
|
+
for item in checks_raw:
|
|
88
|
+
if not isinstance(item, dict) or not {"name", "query"} <= set(item) or set(item) - {"name", "query", "expected", "maxAgeSeconds", "database"}:
|
|
89
|
+
raise DrillError("Each check needs name, query, and one comparison")
|
|
90
|
+
if ("expected" in item) == ("maxAgeSeconds" in item):
|
|
91
|
+
raise DrillError("Each check needs exactly one of expected or maxAgeSeconds")
|
|
92
|
+
name, query, expected = item["name"], item["query"], item.get("expected")
|
|
93
|
+
max_age = item.get("maxAgeSeconds")
|
|
94
|
+
database = item.get("database", "postgres")
|
|
95
|
+
if not isinstance(name, str) or not name or len(name) > 80:
|
|
96
|
+
raise DrillError("Check name must be 1..80 characters")
|
|
97
|
+
if "expected" in item and (not isinstance(expected, str) or len(expected) > 4096):
|
|
98
|
+
raise DrillError("Check expected must be a string up to 4096 characters")
|
|
99
|
+
if "maxAgeSeconds" in item and (type(max_age) is not int or not 1 <= max_age <= 31536000):
|
|
100
|
+
raise DrillError("maxAgeSeconds must be 1..31536000")
|
|
101
|
+
if not isinstance(query, str) or len(query) > 8192 or not re.match(r"^\s*(SELECT|WITH)\b", query, re.I) or ";" in query:
|
|
102
|
+
raise DrillError(f"Check {name}: query must be one SELECT/WITH statement without semicolons")
|
|
103
|
+
if not isinstance(database, str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]{0,62}", database):
|
|
104
|
+
raise DrillError(f"Check {name}: database must be a simple PostgreSQL database name")
|
|
105
|
+
checks.append(Check(name, query.strip(), expected, max_age, database))
|
|
106
|
+
retained = raw.get("retainOnFailure", False)
|
|
107
|
+
if type(retained) is not bool:
|
|
108
|
+
raise DrillError("retainOnFailure must be boolean")
|
|
109
|
+
return cls(namespace, cluster, recovery_object_store, timeout, poll, max_backup_age, target, tuple(checks), retained)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class Kubectl:
|
|
113
|
+
def __init__(self, executable: str = "kubectl", invoke: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run):
|
|
114
|
+
self.executable = executable
|
|
115
|
+
self.invoke = invoke
|
|
116
|
+
|
|
117
|
+
def call(self, args: list[str], *, input_text: str | None = None, timeout: int = 60) -> str:
|
|
118
|
+
try:
|
|
119
|
+
result = self.invoke([self.executable, *args], input=input_text, text=True, capture_output=True, timeout=timeout, check=False)
|
|
120
|
+
except (FileNotFoundError, subprocess.TimeoutExpired) as exc:
|
|
121
|
+
raise DrillError(f"kubectl unavailable or timed out: {exc}") from exc
|
|
122
|
+
if result.returncode:
|
|
123
|
+
raise DrillError(f"kubectl {' '.join(args[:4])} failed: {result.stderr.strip()[:1000]}")
|
|
124
|
+
return result.stdout
|
|
125
|
+
|
|
126
|
+
def get(self, resource: str, name: str, namespace: str) -> dict[str, Any]:
|
|
127
|
+
return json.loads(self.call(["-n", namespace, "get", resource, name, "-o", "json"]))
|
|
128
|
+
|
|
129
|
+
def list_pvcs(self, cluster: str, namespace: str) -> list[str]:
|
|
130
|
+
data = json.loads(self.call(["-n", namespace, "get", "pvc", "-l", f"cnpg.io/cluster={cluster}", "-o", "json"]))
|
|
131
|
+
return [item["metadata"]["name"] for item in data.get("items", [])]
|
|
132
|
+
|
|
133
|
+
def recovery_logs(self, cluster: str, namespace: str) -> list[str]:
|
|
134
|
+
data = json.loads(self.call(["-n", namespace, "get", "pods", "-l", f"cnpg.io/cluster={cluster}", "-o", "json"]))
|
|
135
|
+
logs = []
|
|
136
|
+
pods = sorted(data.get("items", []), key=lambda p: p.get("metadata", {}).get("creationTimestamp", ""), reverse=True)
|
|
137
|
+
for pod in pods[:5]:
|
|
138
|
+
name = pod.get("metadata", {}).get("name", "")
|
|
139
|
+
if "full-recovery" not in name:
|
|
140
|
+
continue
|
|
141
|
+
for container in ("plugin-barman-cloud", "full-recovery"):
|
|
142
|
+
try:
|
|
143
|
+
logs.append(self.call(["-n", namespace, "logs", name, "-c", container, "--tail=200"], timeout=20))
|
|
144
|
+
except DrillError:
|
|
145
|
+
continue
|
|
146
|
+
return logs
|
|
147
|
+
|
|
148
|
+
def list_backups(self, namespace: str) -> list[dict[str, Any]]:
|
|
149
|
+
data = json.loads(self.call(["-n", namespace, "get", "backups.postgresql.cnpg.io", "-o", "json"]))
|
|
150
|
+
return data.get("items", [])
|
|
151
|
+
|
|
152
|
+
def create(self, manifest: dict[str, Any], namespace: str) -> None:
|
|
153
|
+
self.call(["-n", namespace, "create", "-f", "-"], input_text=json.dumps(manifest))
|
|
154
|
+
|
|
155
|
+
def delete(self, resource: str, name: str, namespace: str, *, timeout: int = 120) -> None:
|
|
156
|
+
self.call(["-n", namespace, "delete", resource, name, "--ignore-not-found=true", "--wait=true", f"--timeout={timeout}s"], timeout=timeout + 15)
|
|
157
|
+
|
|
158
|
+
def exec_query(self, pod: str, namespace: str, query: str, database: str = "postgres") -> str:
|
|
159
|
+
# One read-only transaction. The config parser rejects semicolons in query.
|
|
160
|
+
sql = f"BEGIN READ ONLY; {query}; COMMIT;"
|
|
161
|
+
return self.call(["-n", namespace, "exec", pod, "-c", "postgres", "--", "psql", "-X", "-qAt", "-v", "ON_ERROR_STOP=1", "-U", "postgres", "-d", database, "-c", sql], timeout=60).strip()
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def choose_backup(backups: list[dict[str, Any]], config: Config, object_name: str | None = None) -> tuple[dict[str, Any], int]:
|
|
165
|
+
now = dt.datetime.now(dt.timezone.utc)
|
|
166
|
+
target = dt.datetime.fromisoformat(config.target_time.replace("Z", "+00:00")) if config.target_time else now
|
|
167
|
+
candidates: list[tuple[dt.datetime, dict[str, Any]]] = []
|
|
168
|
+
for backup in backups:
|
|
169
|
+
if backup.get("spec", {}).get("cluster", {}).get("name") != config.cluster:
|
|
170
|
+
continue
|
|
171
|
+
status = backup.get("status", {})
|
|
172
|
+
if status.get("phase") != "completed" or not status.get("backupId"):
|
|
173
|
+
continue
|
|
174
|
+
if (status.get("method") or backup.get("spec", {}).get("method")) != "plugin":
|
|
175
|
+
continue
|
|
176
|
+
if backup.get("spec", {}).get("pluginConfiguration", {}).get("name") not in (None, PLUGIN):
|
|
177
|
+
continue
|
|
178
|
+
backup_object = backup.get("spec", {}).get("pluginConfiguration", {}).get("parameters", {}).get("barmanObjectName")
|
|
179
|
+
if object_name and backup_object and backup_object != object_name:
|
|
180
|
+
continue
|
|
181
|
+
try:
|
|
182
|
+
stopped = dt.datetime.fromisoformat(status["stoppedAt"].replace("Z", "+00:00"))
|
|
183
|
+
except (KeyError, TypeError, ValueError):
|
|
184
|
+
continue
|
|
185
|
+
if stopped.tzinfo and stopped <= target:
|
|
186
|
+
candidates.append((stopped, backup))
|
|
187
|
+
if not candidates:
|
|
188
|
+
raise DrillError("No completed Barman plugin Backup with an ID exists before the recovery target")
|
|
189
|
+
stopped, chosen = max(candidates, key=lambda item: item[0])
|
|
190
|
+
age = int((now - stopped).total_seconds())
|
|
191
|
+
if age > config.max_backup_age_seconds:
|
|
192
|
+
raise DrillError(f"Latest eligible Backup is {age}s old, above maxBackupAgeSeconds={config.max_backup_age_seconds}")
|
|
193
|
+
return chosen, age
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def build_manifest(source: dict[str, Any], config: Config, *, name: str, backup: dict[str, Any] | None = None) -> dict[str, Any]:
|
|
197
|
+
spec = source.get("spec", {})
|
|
198
|
+
if spec.get("tablespaces"):
|
|
199
|
+
raise DrillError("Tablespace clusters are not supported in v0.1; refusing incomplete restore")
|
|
200
|
+
bootstrap = spec.get("bootstrap") or {}
|
|
201
|
+
if bootstrap.get("recovery") or bootstrap.get("pg_basebackup"):
|
|
202
|
+
raise DrillError("Replica or recovery-source clusters are not supported in v0.1")
|
|
203
|
+
if bootstrap and "initdb" not in bootstrap:
|
|
204
|
+
raise DrillError("Only default or initdb bootstrap sources are supported in v0.1")
|
|
205
|
+
plugins = [p for p in spec.get("plugins", []) if p.get("name") == PLUGIN and p.get("enabled", True)]
|
|
206
|
+
if len(plugins) != 1:
|
|
207
|
+
raise DrillError("Source needs one enabled Barman Cloud plugin")
|
|
208
|
+
params = plugins[0].get("parameters", {})
|
|
209
|
+
object_name = params.get("barmanObjectName")
|
|
210
|
+
if not isinstance(object_name, str) or not DNS_NAME.fullmatch(object_name):
|
|
211
|
+
raise DrillError("Barman plugin needs a valid barmanObjectName")
|
|
212
|
+
storage = spec.get("storage")
|
|
213
|
+
if not isinstance(storage, dict) or not storage.get("size"):
|
|
214
|
+
raise DrillError("Source Cluster needs storage.size")
|
|
215
|
+
if len(name) > 63 or not DNS_NAME.fullmatch(name) or name == config.cluster:
|
|
216
|
+
raise DrillError("Invalid or unsafe drill cluster name")
|
|
217
|
+
recovery: dict[str, Any] = {"source": "backup-source"}
|
|
218
|
+
if backup:
|
|
219
|
+
recovery["recoveryTarget"] = {"backupID": backup["status"]["backupId"]}
|
|
220
|
+
initdb = bootstrap.get("initdb") or {}
|
|
221
|
+
for key in ("database", "owner", "secret"):
|
|
222
|
+
if key in initdb:
|
|
223
|
+
recovery[key] = copy.deepcopy(initdb[key])
|
|
224
|
+
if config.target_time:
|
|
225
|
+
recovery.setdefault("recoveryTarget", {})["targetTime"] = config.target_time
|
|
226
|
+
external = {"name": "backup-source", "plugin": {"name": PLUGIN, "enabled": True, "parameters": {"barmanObjectName": config.recovery_object_store or object_name, "serverName": params.get("serverName") or config.cluster}}}
|
|
227
|
+
output: dict[str, Any] = {
|
|
228
|
+
"apiVersion": "postgresql.cnpg.io/v1", "kind": "Cluster",
|
|
229
|
+
"metadata": {"name": name, "namespace": config.namespace, "labels": {"app.kubernetes.io/managed-by": "cnpg-drill", "cnpg-drill.dev/source": config.cluster}, "annotations": {"cnpg-drill.dev/source-uid": source.get("metadata", {}).get("uid", ""), "cnpg-drill.dev/backup-name": backup.get("metadata", {}).get("name", "") if backup else ""}},
|
|
230
|
+
"spec": {"instances": 1, "storage": copy.deepcopy(storage), "bootstrap": {"recovery": recovery}, "externalClusters": [external]},
|
|
231
|
+
}
|
|
232
|
+
for key in ("imageName", "imageCatalogRef", "walStorage", "resources"):
|
|
233
|
+
if key in spec:
|
|
234
|
+
output["spec"][key] = copy.deepcopy(spec[key])
|
|
235
|
+
# S3-compatible stores may need these boto3 settings during recovery too.
|
|
236
|
+
# Never copy arbitrary source env: it can contain writer credentials.
|
|
237
|
+
recovery_env = [
|
|
238
|
+
{"name": item["name"], "value": item["value"]} for item in spec.get("env", [])
|
|
239
|
+
if isinstance(item, dict)
|
|
240
|
+
and item.get("name") in {"AWS_REQUEST_CHECKSUM_CALCULATION", "AWS_RESPONSE_CHECKSUM_VALIDATION", "AWS_NO_CHUNKED_ENCODING"}
|
|
241
|
+
and isinstance(item.get("value"), str)
|
|
242
|
+
and "valueFrom" not in item
|
|
243
|
+
]
|
|
244
|
+
if recovery_env:
|
|
245
|
+
output["spec"]["env"] = recovery_env
|
|
246
|
+
# Deliberately do not copy spec.plugins: the recovered cluster must not archive to source.
|
|
247
|
+
return output
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def new_name(source_name: str) -> str:
|
|
251
|
+
suffix = dt.datetime.now(dt.timezone.utc).strftime("%m%d%H%M") + "-" + secrets.token_hex(3)
|
|
252
|
+
return f"drill-{source_name[:48-len(suffix)]}-{suffix}"
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def recovery_failure_reason(logs: list[str]) -> str | None:
|
|
256
|
+
"""Classify recovery logs without exposing archive paths, credentials, or SQL."""
|
|
257
|
+
messages = []
|
|
258
|
+
for log in logs:
|
|
259
|
+
for line in log.splitlines()[-200:]:
|
|
260
|
+
try:
|
|
261
|
+
entry = json.loads(line)
|
|
262
|
+
except json.JSONDecodeError:
|
|
263
|
+
continue
|
|
264
|
+
if not isinstance(entry, dict):
|
|
265
|
+
continue
|
|
266
|
+
record = entry.get("record")
|
|
267
|
+
if isinstance(record, dict):
|
|
268
|
+
messages.append(str(record.get("message", "")))
|
|
269
|
+
messages.append(str(entry.get("error", "")))
|
|
270
|
+
messages.append(str(entry.get("msg", "")))
|
|
271
|
+
combined = "\n".join(messages).lower()
|
|
272
|
+
if re.search(r"access.?denied|permission.?denied|forbidden|\b403\b", combined):
|
|
273
|
+
return "archive_access_denied"
|
|
274
|
+
if re.search(r"(wal|archive).{0,100}(not found|no such key|missing|unavailable|does not exist)|requested wal segment.{0,100}removed", combined):
|
|
275
|
+
return "wal_unavailable"
|
|
276
|
+
if re.search(r"restore error|error while restoring a backup|fatal", combined):
|
|
277
|
+
return "recovery_process_failed"
|
|
278
|
+
return None
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def prepare(client: Kubectl, config: Config, name: str | None = None) -> dict[str, Any]:
|
|
282
|
+
source = client.get(API, config.cluster, config.namespace)
|
|
283
|
+
if source.get("metadata", {}).get("deletionTimestamp"):
|
|
284
|
+
raise DrillError("Source Cluster is being deleted")
|
|
285
|
+
source_plugins = [p for p in source.get("spec", {}).get("plugins", []) if p.get("name") == PLUGIN and p.get("enabled", True)]
|
|
286
|
+
object_name = source_plugins[0].get("parameters", {}).get("barmanObjectName") if len(source_plugins) == 1 else None
|
|
287
|
+
backup, age = choose_backup(client.list_backups(config.namespace), config, object_name)
|
|
288
|
+
manifest = build_manifest(source, config, name=name or new_name(config.cluster), backup=backup)
|
|
289
|
+
manifest["metadata"]["annotations"]["cnpg-drill.dev/backup-age-seconds"] = str(age)
|
|
290
|
+
if not object_name:
|
|
291
|
+
raise DrillError("Source needs one enabled Barman Cloud plugin")
|
|
292
|
+
source_store = client.get("objectstores.barmancloud.cnpg.io", object_name, config.namespace)
|
|
293
|
+
recovery_name = config.recovery_object_store or object_name
|
|
294
|
+
if recovery_name != object_name:
|
|
295
|
+
recovery_store = client.get("objectstores.barmancloud.cnpg.io", recovery_name, config.namespace)
|
|
296
|
+
source_config = source_store.get("spec", {}).get("configuration", {})
|
|
297
|
+
recovery_config = recovery_store.get("spec", {}).get("configuration", {})
|
|
298
|
+
for field in ("destinationPath", "endpointURL"):
|
|
299
|
+
if not source_config.get(field) or source_config.get(field) != recovery_config.get(field):
|
|
300
|
+
raise DrillError(f"Recovery ObjectStore {field} must match source ObjectStore")
|
|
301
|
+
return manifest
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def run_drill(client: Kubectl, config: Config, *, sleep: Callable[[float], None] = time.sleep, monotonic: Callable[[], float] = time.monotonic) -> dict[str, Any]:
|
|
305
|
+
started_at = dt.datetime.now(dt.timezone.utc)
|
|
306
|
+
started = monotonic()
|
|
307
|
+
result: dict[str, Any] = {"schemaVersion": 1, "source": f"{config.namespace}/{config.cluster}", "startedAt": started_at.isoformat(), "targetTime": config.target_time, "checks": [], "status": "error"}
|
|
308
|
+
name: str | None = None
|
|
309
|
+
created = False
|
|
310
|
+
try:
|
|
311
|
+
manifest = prepare(client, config)
|
|
312
|
+
name = manifest["metadata"]["name"]
|
|
313
|
+
result["drillCluster"] = name
|
|
314
|
+
result["sourceClusterUID"] = manifest["metadata"]["annotations"]["cnpg-drill.dev/source-uid"]
|
|
315
|
+
result["backupName"] = manifest["metadata"]["annotations"]["cnpg-drill.dev/backup-name"]
|
|
316
|
+
result["backupID"] = manifest["spec"]["bootstrap"]["recovery"]["recoveryTarget"]["backupID"]
|
|
317
|
+
result["backupAgeSeconds"] = int(manifest["metadata"]["annotations"]["cnpg-drill.dev/backup-age-seconds"])
|
|
318
|
+
created = True
|
|
319
|
+
# A create request can reach the API even if the client loses its response.
|
|
320
|
+
# In that case, still try to remove the uniquely named drill Cluster.
|
|
321
|
+
client.create(manifest, config.namespace)
|
|
322
|
+
recovery_started = monotonic()
|
|
323
|
+
deadline = started + config.timeout_seconds
|
|
324
|
+
primary = None
|
|
325
|
+
while monotonic() < deadline:
|
|
326
|
+
cluster = client.get(API, name, config.namespace)
|
|
327
|
+
status = cluster.get("status", {})
|
|
328
|
+
if status.get("readyInstances", 0) >= 1 and status.get("currentPrimary"):
|
|
329
|
+
primary = status["currentPrimary"]
|
|
330
|
+
break
|
|
331
|
+
sleep(config.poll_seconds)
|
|
332
|
+
if not primary:
|
|
333
|
+
try:
|
|
334
|
+
reason = recovery_failure_reason(client.recovery_logs(name, config.namespace))
|
|
335
|
+
except DrillError:
|
|
336
|
+
reason = None
|
|
337
|
+
result["failureReason"] = reason or "recovery_timeout"
|
|
338
|
+
hints = {
|
|
339
|
+
"archive_access_denied": "Check recovery ObjectStore permissions to read both base backup and WAL objects.",
|
|
340
|
+
"wal_unavailable": "Check that the required WAL segment exists and is readable in the recovery archive.",
|
|
341
|
+
"recovery_process_failed": "Inspect the drill recovery Pod and operator logs for the underlying restore error.",
|
|
342
|
+
"recovery_timeout": "Inspect the drill recovery Pod and operator logs for the cause.",
|
|
343
|
+
}
|
|
344
|
+
raise DrillError(f"Recovery did not become ready within {config.timeout_seconds}s. {hints[result['failureReason']]}")
|
|
345
|
+
result["recoverySeconds"] = round(monotonic() - recovery_started, 2)
|
|
346
|
+
for check in config.checks:
|
|
347
|
+
observed = client.exec_query(primary, config.namespace, check.query, check.database)
|
|
348
|
+
check_result: dict[str, Any] = {"name": check.name, "database": check.database, "observedSha256": hashlib.sha256(observed.encode()).hexdigest()}
|
|
349
|
+
if check.max_age_seconds is not None:
|
|
350
|
+
try:
|
|
351
|
+
observed_time = dt.datetime.fromisoformat(observed.replace("Z", "+00:00"))
|
|
352
|
+
if observed_time.tzinfo is None:
|
|
353
|
+
raise ValueError("timestamp lacks timezone")
|
|
354
|
+
age = (dt.datetime.now(dt.timezone.utc) - observed_time).total_seconds()
|
|
355
|
+
check_result["ageSeconds"] = round(age, 2)
|
|
356
|
+
check_result["maxAgeSeconds"] = check.max_age_seconds
|
|
357
|
+
passed = 0 <= age <= check.max_age_seconds
|
|
358
|
+
except ValueError:
|
|
359
|
+
passed = False
|
|
360
|
+
check_result["error"] = "query did not return a timestamp with timezone"
|
|
361
|
+
else:
|
|
362
|
+
passed = observed == check.expected
|
|
363
|
+
check_result["passed"] = passed
|
|
364
|
+
result["checks"].append(check_result)
|
|
365
|
+
if not passed:
|
|
366
|
+
raise DrillError(f"Check {check.name!r} did not meet its assertion")
|
|
367
|
+
result["status"] = "passed"
|
|
368
|
+
except (DrillError, ValueError, KeyError) as exc:
|
|
369
|
+
result["error"] = str(exc)
|
|
370
|
+
result["status"] = "failed"
|
|
371
|
+
if not created:
|
|
372
|
+
result["phase"] = "preflight"
|
|
373
|
+
finally:
|
|
374
|
+
if created and name and not (result["status"] == "failed" and config.retain_on_failure):
|
|
375
|
+
try:
|
|
376
|
+
client.delete(API, name, config.namespace)
|
|
377
|
+
cleanup_deadline = monotonic() + 60
|
|
378
|
+
remaining = client.list_pvcs(name, config.namespace)
|
|
379
|
+
while remaining and monotonic() < cleanup_deadline:
|
|
380
|
+
sleep(2)
|
|
381
|
+
remaining = client.list_pvcs(name, config.namespace)
|
|
382
|
+
if remaining:
|
|
383
|
+
raise DrillError(f"Drill PVCs still exist after Cluster deletion: {', '.join(remaining)}")
|
|
384
|
+
result["cleanup"] = "cluster-and-pvcs-deleted"
|
|
385
|
+
except DrillError as exc:
|
|
386
|
+
result["cleanup"] = "failed"
|
|
387
|
+
result["cleanupError"] = str(exc)
|
|
388
|
+
result["status"] = "failed"
|
|
389
|
+
elif created:
|
|
390
|
+
result["cleanup"] = "retained-for-investigation"
|
|
391
|
+
result["finishedAt"] = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
392
|
+
result["durationSeconds"] = round(monotonic() - started, 2)
|
|
393
|
+
return result
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cnpg-drill
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Prove CloudNativePG Barman backups restore into an isolated cluster
|
|
5
|
+
Author: cnpg-drill contributors
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2026 cnpg-drill contributors
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Keywords: cloudnativepg,postgresql,kubernetes,backup,restore
|
|
29
|
+
Classifier: Programming Language :: Python :: 3
|
|
30
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
31
|
+
Classifier: Operating System :: OS Independent
|
|
32
|
+
Requires-Python: >=3.10
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
License-File: LICENSE
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# cnpg-drill
|
|
38
|
+
|
|
39
|
+
**Prove that a CloudNativePG backup can restore.** `cnpg-drill` creates a disposable CloudNativePG cluster from the source cluster's Barman Cloud object store, waits for PostgreSQL to become ready, runs read-only SQL assertions, emits a JSON result, and deletes the drill cluster. It can run locally or on a schedule as a Kubernetes CronJob.
|
|
40
|
+
|
|
41
|
+
The core is free and works without an account or external service. It does not make backups or copy database contents to a vendor. The proposed hosted service in [RESEARCH.md](RESEARCH.md) is separate and has not been built.
|
|
42
|
+
|
|
43
|
+
## Status
|
|
44
|
+
|
|
45
|
+
**Early alpha.** Unit tests cover manifest safety and execution state. A disposable [local integration run](integration/local/README.md) on v0.1.0 proved full restore, PITR, read-only archive access, denied-WAL reporting, failed-check reporting, and Cluster/PVC cleanup on one version matrix. The v0.1.1 release archive and public chart 0.1.2 also restored a fresh backup and checked a row in an application database. A separate single-node pgEdge chart restore passed SQL checks in its `app` database. Validate against your operator, PostgreSQL image, storage, object store, and Barman plugin versions before treating a pass as disaster recovery assurance.
|
|
46
|
+
|
|
47
|
+
The [v0.1.2 release](https://github.com/danielgaskins/cnpg-drill/releases/tag/v0.1.2) carries literal AWS checksum compatibility settings into recovery clusters without copying source credentials. It includes the MIT-licensed kubectl plugin archive and checksum. The [default Krew index submission](https://github.com/kubernetes-sigs/krew-index/pull/6373) and [custom Krew index submission](https://github.com/ishantanu/awesome-kubectl-plugins/pull/44) currently package v0.1.0 and are under review. [kubetools](https://github.com/collabnix/kubetools/pull/431) is reviewing a Backup Tools listing.
|
|
48
|
+
|
|
49
|
+
The [Helm chart is listed on Artifact Hub](https://artifacthub.io/packages/helm/cnpg-drill/cnpg-drill) as a Verified Publisher package. Chart version `0.1.2` uses the v0.1.1 application image and includes the publisher's [website](https://danielgaskins.com/).
|
|
50
|
+
|
|
51
|
+
## What it supports
|
|
52
|
+
|
|
53
|
+
- One source CloudNativePG cluster with one enabled Barman Cloud plugin and a named `ObjectStore` in the same namespace.
|
|
54
|
+
- An optional separate `recoveryObjectStore` with credentials limited to archive reads. Preflight checks that its destination and endpoint match the source store before creating a Cluster.
|
|
55
|
+
- Full restore from the latest completed Barman plugin `Backup` resource, or PITR to an explicit RFC3339 `targetTime`. The selected backup ID is pinned and included in the report; a stale backup fails preflight.
|
|
56
|
+
- Single-instance drill cluster using the source's PostgreSQL image, storage size/class, optional WAL storage, and resource requests.
|
|
57
|
+
- Copies literal AWS checksum compatibility settings from the source Cluster for S3-compatible archive restores; it does not copy writer credentials or arbitrary environment variables.
|
|
58
|
+
- Read-only SQL checks in `postgres` or a named application database. Exact expected values make reports easy to audit.
|
|
59
|
+
- Optional timestamp freshness assertions (`maxAgeSeconds`) for an application-defined recovery point. This measures age of that application's data, not the database WAL RPO.
|
|
60
|
+
- A JSON report on stdout and optionally in a local file. Query outputs are hashed rather than logged. Exit code 0 means all checks, Cluster deletion, and PVC cleanup passed; 1 means the drill failed; 2 means invalid input or a failed preflight.
|
|
61
|
+
- Scheduled execution with the Helm CronJob in `deploy/helm/cnpg-drill`.
|
|
62
|
+
|
|
63
|
+
The CLI refuses source clusters with tablespaces or recovery bootstrap because a naive clone could produce an incomplete or misleading pass. Volume snapshots, cross-namespace restores, cross-region recovery, custom Postgres extension image changes, and multi-cluster fleet management are future work.
|
|
64
|
+
|
|
65
|
+
## Install and use
|
|
66
|
+
|
|
67
|
+
For a first run on your own cluster, follow the [first-run guide](docs/FIRST-RUN.md).
|
|
68
|
+
|
|
69
|
+
Requirements: Python 3.10+, `kubectl` in `PATH`, CloudNativePG and the Barman Cloud plugin installed in the target cluster, and Kubernetes access to read the source Cluster and ObjectStore, create/get/delete a drill Cluster, and exec into its PostgreSQL pod.
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
python3 -m pip install -e .
|
|
73
|
+
cnpg-drill plan --config examples/drill.json
|
|
74
|
+
cnpg-drill run --config examples/drill.json --report reports/latest.json
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
`plan` reads the source configuration and prints the exact Cluster manifest it would create. Review that manifest before running the first drill. The example's table-count assertion is illustrative; replace it with an invariant from your own application. A minimal check is `SELECT 1`, but it only proves connection to the recovered server, not useful application data.
|
|
78
|
+
|
|
79
|
+
Example `drill.json`:
|
|
80
|
+
|
|
81
|
+
```json
|
|
82
|
+
{
|
|
83
|
+
"namespace": "production",
|
|
84
|
+
"cluster": "app-db",
|
|
85
|
+
"recoveryObjectStore": "app-db-recovery-readonly",
|
|
86
|
+
"timeoutSeconds": 1800,
|
|
87
|
+
"maxBackupAgeSeconds": 691200,
|
|
88
|
+
"checks": [
|
|
89
|
+
{"name": "postgres-ready", "query": "SELECT 1", "expected": "1"},
|
|
90
|
+
{"name": "orders-exist", "database": "app", "query": "SELECT count(*) > 0 FROM public.orders", "expected": "t"},
|
|
91
|
+
{"name": "recent-order", "database": "app", "query": "SELECT max(created_at)::timestamptz FROM public.orders", "maxAgeSeconds": 86400}
|
|
92
|
+
]
|
|
93
|
+
}
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
For PITR, add `"targetTime": "2026-09-29T10:00:00Z"`. Pick a time inside the backup and WAL retention window. A passing PITR check proves only that target; use a second drill for the latest recovery path.
|
|
97
|
+
|
|
98
|
+
## Scheduled drill
|
|
99
|
+
|
|
100
|
+
The Helm chart creates a namespace-scoped ServiceAccount, Role, RoleBinding, ConfigMap, and CronJob. Set `cluster`, `checks`, and the other settings in a values file and install it **in the same namespace as the source Cluster and ObjectStore**. The chart defaults to suspension so installation does not immediately start a restore.
|
|
101
|
+
|
|
102
|
+
The example image build pins kubectl 1.36.4, suitable for Kubernetes 1.35–1.37 under the project's [version skew policy](https://kubernetes.io/releases/). Set `KUBECTL_VERSION` at image build time for another supported cluster version.
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
helm install cnpg-drill oci://ghcr.io/danielgaskins/charts/cnpg-drill \
|
|
106
|
+
--version 0.1.2 -n production -f drill-values.yaml
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
See the [first-run guide](docs/FIRST-RUN.md) and [chart values](deploy/helm/cnpg-drill/README.md) for a suspended first run with a separate read-only recovery ObjectStore. The chart pins a public multi-architecture image digest. The CronJob uses `concurrencyPolicy: Forbid` and a bounded job deadline. Its logs contain the JSON result; failed runs have nonzero exit status. **The chart does not yet provide durable report storage, missed-run alerts, or fleet policy.**
|
|
110
|
+
|
|
111
|
+
## Safety boundary
|
|
112
|
+
|
|
113
|
+
- The generated recovery Cluster never copies `spec.plugins`, so it is not configured to archive WAL back to the source bucket. Set `recoveryObjectStore` to a separately named `ObjectStore` with read-only archive credentials. The tool compares its destination and endpoint with the source store; it cannot verify credential permissions, so test that writes are denied before relying on this boundary. Omitting this field uses the source `ObjectStore` for compatibility.
|
|
114
|
+
- A recovery timeout includes a machine-readable `failureReason` when the recovery Pod logs identify archive access denial or unavailable WAL. It does not copy raw logs into the report; inspect Pod and operator logs for detail.
|
|
115
|
+
- The drill uses the same namespace because the plugin's `ObjectStore` and its credentials are namespace-scoped. The source Cluster object is never modified.
|
|
116
|
+
- SQL checks run in a `BEGIN READ ONLY` transaction. Queries are limited to one `SELECT` or `WITH` statement without semicolons. Only provide trusted SQL; this is a guard against mistakes, not a security sandbox.
|
|
117
|
+
- By default, the tool deletes the drill Cluster even when recovery or a check fails. `retainOnFailure` leaves it for investigation and can incur storage costs. Cleanup failures turn the result red.
|
|
118
|
+
- The tool checks that PVCs labeled for the drill cluster disappear after Cluster deletion and fails the report if they remain. Kubernetes PersistentVolume cleanup depends on the storage class; inspect cloud volumes after a first drill.
|
|
119
|
+
- The planned SaaS boundary is metadata only. This repository contains no telemetry upload or hosted service.
|
|
120
|
+
|
|
121
|
+
## Contributing and upstream distribution
|
|
122
|
+
|
|
123
|
+
Run `PYTHONPATH=src python3 -m unittest discover -s tests -v`. See [CONTRIBUTING.md](CONTRIBUTING.md) for how to add a recovery path, the live integration test gate, and the Krew package plan. We will submit only a tested, released package that stands on its own. Upstream acceptance is not assumed.
|
|
124
|
+
|
|
125
|
+
[UPSTREAM.md](UPSTREAM.md) records the specific repositories, proposed contribution value, and submission gates.
|
|
126
|
+
|
|
127
|
+
## Recovery references
|
|
128
|
+
|
|
129
|
+
- [CloudNativePG backup guidance](https://cloudnative-pg.io/docs/devel/backup/)
|
|
130
|
+
- [CloudNativePG recovery documentation](https://cloudnative-pg.io/docs/devel/recovery/)
|
|
131
|
+
- [Barman Cloud recovery example](https://cloudnative-pg.io/plugin-barman-cloud/docs/next/concepts/)
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
cnpg_drill/__init__.py,sha256=bsuuJxSuvjphv2BC76ueuv60gNLgX98OvIBvJhBN1iE,59
|
|
2
|
+
cnpg_drill/cli.py,sha256=Gq9kprNdYsyq7TQoDF8nGme3Iyg-MqJ0bPTwCbrfJSc,1784
|
|
3
|
+
cnpg_drill/core.py,sha256=6kf97ZtiTogYg2g8N6FdOrY59GTfYTpYtaIGQOqr1jM,22322
|
|
4
|
+
cnpg_drill-0.1.2.dist-info/licenses/LICENSE,sha256=tp91pBoNJGc9mfBSenjbIuE4t_TAji0ohKLZ9kq3yek,1080
|
|
5
|
+
cnpg_drill-0.1.2.dist-info/METADATA,sha256=AX4vSljB5aPQ6b4HPc8QMGci4O9zAp_TlRw6r9Y1FfE,10869
|
|
6
|
+
cnpg_drill-0.1.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
7
|
+
cnpg_drill-0.1.2.dist-info/entry_points.txt,sha256=4JZjskQhNDiOSIp92Ac4ogdr2oYbjlq-Vv7oqy16CXs,51
|
|
8
|
+
cnpg_drill-0.1.2.dist-info/top_level.txt,sha256=qy7X-k6OPJbOMahO_v-WvV_LUkFBmPFkIVaownyDXBo,11
|
|
9
|
+
cnpg_drill-0.1.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 cnpg-drill contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cnpg_drill
|