nodusdb 0.1.0__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nodusdb/__init__.py +88 -0
- nodusdb/_native.py +162 -0
- nodusdb/_tokens.py +8 -0
- nodusdb/_version.py +6 -0
- nodusdb/bin/windows-x86_64/libnodusdb.dll +4 -0
- nodusdb/errors.py +110 -0
- nodusdb/follower.py +186 -0
- nodusdb/graph.py +977 -0
- nodusdb/history.py +36 -0
- nodusdb/lake.py +396 -0
- nodusdb/shipping.py +121 -0
- nodusdb/stores.py +21 -0
- nodusdb/upgrade.py +43 -0
- nodusdb/watch.py +132 -0
- nodusdb-0.1.0.dist-info/METADATA +90 -0
- nodusdb-0.1.0.dist-info/RECORD +21 -0
- nodusdb-0.1.0.dist-info/WHEEL +5 -0
- nodusdb-0.1.0.dist-info/licenses/LICENSE +202 -0
- nodusdb-0.1.0.dist-info/licenses/NOTICE +11 -0
- nodusdb-0.1.0.dist-info/licenses/licenses/graalvm-native-image.txt +379 -0
- nodusdb-0.1.0.dist-info/top_level.txt +1 -0
nodusdb/__init__.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
from ._native import find_library
|
|
2
|
+
from ._version import __version__
|
|
3
|
+
from .errors import (
|
|
4
|
+
NodusChainTrustError,
|
|
5
|
+
NodusCheckDepthError,
|
|
6
|
+
NodusCorruptLogError,
|
|
7
|
+
NodusError,
|
|
8
|
+
NodusErasurePendingError,
|
|
9
|
+
NodusHistoryExpiredError,
|
|
10
|
+
NodusIndeterminateError,
|
|
11
|
+
NodusLogBacklogError,
|
|
12
|
+
NodusMemoryError,
|
|
13
|
+
NodusSchemaError,
|
|
14
|
+
NodusShipTimeoutError,
|
|
15
|
+
NodusStaleReadError,
|
|
16
|
+
NodusTokenLostError,
|
|
17
|
+
NodusUnsupportedError,
|
|
18
|
+
NodusUpgradeRequiredError,
|
|
19
|
+
NodusWriterFencedError,
|
|
20
|
+
)
|
|
21
|
+
from .follower import (
|
|
22
|
+
Follower,
|
|
23
|
+
Restored,
|
|
24
|
+
SalvagedChange,
|
|
25
|
+
SalvagedTransaction,
|
|
26
|
+
SalvageReport,
|
|
27
|
+
TakeoverReport,
|
|
28
|
+
restore,
|
|
29
|
+
salvage,
|
|
30
|
+
)
|
|
31
|
+
from .graph import Graph, PastGraph, Token, Transaction
|
|
32
|
+
from .lake import LakeTable, key_hash
|
|
33
|
+
from .shipping import (
|
|
34
|
+
Shipping,
|
|
35
|
+
environment_credentials,
|
|
36
|
+
file_credentials,
|
|
37
|
+
generate_signing_key,
|
|
38
|
+
static_credentials,
|
|
39
|
+
)
|
|
40
|
+
from .upgrade import UpgradeReport, upgrade, upgrade_cleanup
|
|
41
|
+
from .watch import AccessChange, AccessWatch, Change, ChangeBatch
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"__version__",
|
|
45
|
+
"AccessChange",
|
|
46
|
+
"AccessWatch",
|
|
47
|
+
"Change",
|
|
48
|
+
"ChangeBatch",
|
|
49
|
+
"Follower",
|
|
50
|
+
"Graph",
|
|
51
|
+
"LakeTable",
|
|
52
|
+
"NodusChainTrustError",
|
|
53
|
+
"NodusCheckDepthError",
|
|
54
|
+
"NodusCorruptLogError",
|
|
55
|
+
"NodusError",
|
|
56
|
+
"NodusErasurePendingError",
|
|
57
|
+
"NodusHistoryExpiredError",
|
|
58
|
+
"NodusIndeterminateError",
|
|
59
|
+
"NodusLogBacklogError",
|
|
60
|
+
"NodusMemoryError",
|
|
61
|
+
"NodusSchemaError",
|
|
62
|
+
"NodusShipTimeoutError",
|
|
63
|
+
"NodusStaleReadError",
|
|
64
|
+
"NodusTokenLostError",
|
|
65
|
+
"NodusUnsupportedError",
|
|
66
|
+
"NodusUpgradeRequiredError",
|
|
67
|
+
"NodusWriterFencedError",
|
|
68
|
+
"PastGraph",
|
|
69
|
+
"Restored",
|
|
70
|
+
"SalvageReport",
|
|
71
|
+
"SalvagedChange",
|
|
72
|
+
"SalvagedTransaction",
|
|
73
|
+
"Shipping",
|
|
74
|
+
"TakeoverReport",
|
|
75
|
+
"Token",
|
|
76
|
+
"Transaction",
|
|
77
|
+
"UpgradeReport",
|
|
78
|
+
"environment_credentials",
|
|
79
|
+
"file_credentials",
|
|
80
|
+
"find_library",
|
|
81
|
+
"generate_signing_key",
|
|
82
|
+
"key_hash",
|
|
83
|
+
"restore",
|
|
84
|
+
"salvage",
|
|
85
|
+
"static_credentials",
|
|
86
|
+
"upgrade",
|
|
87
|
+
"upgrade_cleanup",
|
|
88
|
+
]
|
nodusdb/_native.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import ctypes
|
|
2
|
+
import functools
|
|
3
|
+
import os
|
|
4
|
+
import platform
|
|
5
|
+
import sys
|
|
6
|
+
import threading
|
|
7
|
+
import warnings
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from .errors import NodusError, NodusMemoryError, error_for # noqa: F401
|
|
11
|
+
|
|
12
|
+
ENVIRONMENT_VARIABLE = "NODUSDB_LIBRARY"
|
|
13
|
+
|
|
14
|
+
ERROR = -1
|
|
15
|
+
MEMORY_LIMIT = -3
|
|
16
|
+
|
|
17
|
+
PACKAGE_DIRECTORY = Path(__file__).resolve().parent
|
|
18
|
+
REPOSITORY_ROOT = PACKAGE_DIRECTORY.parents[1]
|
|
19
|
+
|
|
20
|
+
HANDLE = ctypes.c_void_p
|
|
21
|
+
THREAD = ctypes.c_void_p
|
|
22
|
+
NODE = ctypes.c_int64
|
|
23
|
+
RESULT_BUFFER = ctypes.POINTER(ctypes.c_int64)
|
|
24
|
+
|
|
25
|
+
_OPERATING_SYSTEMS = {"Windows": "windows", "Darwin": "macos", "Linux": "linux"}
|
|
26
|
+
_ARCHITECTURES = {"amd64": "x86_64", "x86_64": "x86_64", "arm64": "arm64", "aarch64": "arm64"}
|
|
27
|
+
|
|
28
|
+
_loaded = {}
|
|
29
|
+
_attached = threading.local()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def platform_tag():
|
|
33
|
+
system = platform.system()
|
|
34
|
+
operating_system = _OPERATING_SYSTEMS.get(system, system.lower())
|
|
35
|
+
machine = platform.machine().lower()
|
|
36
|
+
architecture = _ARCHITECTURES.get(machine, machine)
|
|
37
|
+
return f"{operating_system}-{architecture}"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _library_file_names():
|
|
41
|
+
if sys.platform == "win32":
|
|
42
|
+
return ("libnodusdb.dll", "nodusdb.dll")
|
|
43
|
+
if sys.platform == "darwin":
|
|
44
|
+
return ("libnodusdb.dylib",)
|
|
45
|
+
return ("libnodusdb.so",)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _search_directories():
|
|
49
|
+
return (
|
|
50
|
+
PACKAGE_DIRECTORY / "bin" / platform_tag(),
|
|
51
|
+
REPOSITORY_ROOT / "target" / "native",
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def find_library(explicit_path=None):
|
|
56
|
+
if explicit_path is not None:
|
|
57
|
+
candidate = Path(explicit_path)
|
|
58
|
+
if candidate.is_file():
|
|
59
|
+
return candidate
|
|
60
|
+
raise NodusError(f"native library not found: {candidate}")
|
|
61
|
+
|
|
62
|
+
from_environment = os.environ.get(ENVIRONMENT_VARIABLE)
|
|
63
|
+
if from_environment:
|
|
64
|
+
return find_library(from_environment)
|
|
65
|
+
|
|
66
|
+
for directory in _search_directories():
|
|
67
|
+
for name in _library_file_names():
|
|
68
|
+
candidate = directory / name
|
|
69
|
+
if candidate.is_file():
|
|
70
|
+
return candidate
|
|
71
|
+
|
|
72
|
+
raise NodusError(
|
|
73
|
+
"libnodusdb not found; build it with `mvn -Pnative -pl core package` "
|
|
74
|
+
f"or set {ENVIRONMENT_VARIABLE} to the library path"
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _create_isolate(library):
|
|
79
|
+
create = library.graal_create_isolate
|
|
80
|
+
create.restype = ctypes.c_int
|
|
81
|
+
create.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_void_p)]
|
|
82
|
+
isolate = ctypes.c_void_p()
|
|
83
|
+
thread = ctypes.c_void_p()
|
|
84
|
+
if create(None, ctypes.byref(isolate), ctypes.byref(thread)) != 0 or not isolate.value:
|
|
85
|
+
raise NodusError("failed to create the GraalVM isolate")
|
|
86
|
+
return isolate
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def serialized(method):
|
|
90
|
+
@functools.wraps(method)
|
|
91
|
+
def locked(self, *args, **kwargs):
|
|
92
|
+
with self._lock:
|
|
93
|
+
return method(self, *args, **kwargs)
|
|
94
|
+
return locked
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def current_thread(library, isolate):
|
|
98
|
+
"""Return this OS thread's isolate thread, attaching it on first use."""
|
|
99
|
+
attached = _attached.__dict__.setdefault("threads", {})
|
|
100
|
+
thread = attached.get(isolate.value)
|
|
101
|
+
if thread is None:
|
|
102
|
+
attach = library.graal_attach_thread
|
|
103
|
+
attach.restype = ctypes.c_int
|
|
104
|
+
attach.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
|
|
105
|
+
candidate = ctypes.c_void_p()
|
|
106
|
+
if attach(isolate, ctypes.byref(candidate)) != 0 or not candidate.value:
|
|
107
|
+
raise NodusError("failed to attach this thread to the GraalVM isolate")
|
|
108
|
+
thread = candidate
|
|
109
|
+
attached[isolate.value] = thread
|
|
110
|
+
return thread
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def last_error_message(library, thread):
|
|
114
|
+
"""The message of the failure this thread's last native call reported."""
|
|
115
|
+
function = library.nodus_last_error
|
|
116
|
+
function.restype = ctypes.c_int
|
|
117
|
+
function.argtypes = [THREAD, ctypes.c_char_p, ctypes.c_int]
|
|
118
|
+
buffer = ctypes.create_string_buffer(512)
|
|
119
|
+
length = function(thread, buffer, len(buffer))
|
|
120
|
+
if length > len(buffer):
|
|
121
|
+
buffer = ctypes.create_string_buffer(length)
|
|
122
|
+
length = function(thread, buffer, len(buffer))
|
|
123
|
+
return buffer.raw[:max(length, 0)].decode("utf-8", "replace")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class OwnedHandle:
|
|
127
|
+
"""A native handle and the call that releases it, shared by its owner and the owner's finalizer.
|
|
128
|
+
|
|
129
|
+
It must not refer to the owner, or the finalizer would keep the owner alive."""
|
|
130
|
+
|
|
131
|
+
def __init__(self, library, isolate, value, release, failure):
|
|
132
|
+
self._library = library
|
|
133
|
+
self._isolate = isolate
|
|
134
|
+
self._release = release
|
|
135
|
+
self._failure = failure
|
|
136
|
+
self.value = value
|
|
137
|
+
|
|
138
|
+
def close(self):
|
|
139
|
+
if not self.value:
|
|
140
|
+
return
|
|
141
|
+
if not self._release(current_thread(self._library, self._isolate), self.value):
|
|
142
|
+
raise NodusError(self._failure)
|
|
143
|
+
self.value = None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def release_unclosed(owned, kind):
|
|
147
|
+
"""Finalizer body: free a handle whose owner was collected without close()."""
|
|
148
|
+
if not owned.value:
|
|
149
|
+
return
|
|
150
|
+
warnings.warn(f"unclosed nodusdb {kind}; call close() or use a with block", ResourceWarning, stacklevel=2)
|
|
151
|
+
try:
|
|
152
|
+
owned.close()
|
|
153
|
+
except NodusError as error:
|
|
154
|
+
warnings.warn(f"nodusdb {kind} could not be closed: {error}", ResourceWarning, stacklevel=2)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def load(explicit_path=None):
|
|
158
|
+
path = str(find_library(explicit_path))
|
|
159
|
+
if path not in _loaded:
|
|
160
|
+
library = ctypes.CDLL(path)
|
|
161
|
+
_loaded[path] = (library, _create_isolate(library))
|
|
162
|
+
return _loaded[path]
|
nodusdb/_tokens.py
ADDED
nodusdb/_version.py
ADDED
nodusdb/errors.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
class NodusError(RuntimeError):
|
|
2
|
+
code = -1
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class NodusMemoryError(NodusError, MemoryError):
|
|
6
|
+
"""The graph's native memory limit would be exceeded. Existing data is unchanged."""
|
|
7
|
+
|
|
8
|
+
code = -3
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class NodusStaleReadError(NodusError):
|
|
12
|
+
"""The graph has not yet applied the write the token names."""
|
|
13
|
+
|
|
14
|
+
code = -4
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class NodusCheckDepthError(NodusError):
|
|
18
|
+
"""A check reached its depth limit before it could decide."""
|
|
19
|
+
|
|
20
|
+
code = -5
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class NodusSchemaError(NodusError):
|
|
24
|
+
"""The schema or a tuple violates the schema. Nothing changed."""
|
|
25
|
+
|
|
26
|
+
code = -6
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class NodusLogBacklogError(NodusError):
|
|
30
|
+
"""The unshipped log reached its cap."""
|
|
31
|
+
|
|
32
|
+
code = -7
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class NodusWriterFencedError(NodusError):
|
|
36
|
+
"""A newer writer exists for this graph."""
|
|
37
|
+
|
|
38
|
+
code = -8
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class NodusCorruptLogError(NodusError):
|
|
42
|
+
"""The log is damaged inside the part known to be durable."""
|
|
43
|
+
|
|
44
|
+
code = -9
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class NodusUnsupportedError(NodusError):
|
|
48
|
+
"""The operation is not available in this mode or version."""
|
|
49
|
+
|
|
50
|
+
code = -10
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class NodusUpgradeRequiredError(NodusError):
|
|
54
|
+
"""The directory was written by an earlier version; run upgrade() first."""
|
|
55
|
+
|
|
56
|
+
code = -11
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class NodusIndeterminateError(NodusError):
|
|
60
|
+
"""A log force failed, so the write may or may not have survived; reopen the graph to find out."""
|
|
61
|
+
|
|
62
|
+
code = -12
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class NodusTokenLostError(NodusError):
|
|
66
|
+
"""The write the token names was acknowledged but never became durable."""
|
|
67
|
+
|
|
68
|
+
code = -13
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class NodusShipTimeoutError(NodusError):
|
|
72
|
+
"""The write is applied and locally durable but had not reached the object store when the wait ended."""
|
|
73
|
+
|
|
74
|
+
code = -14
|
|
75
|
+
token = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class NodusChainTrustError(NodusError):
|
|
79
|
+
"""The object store holds chain data that fails verification: a bad signature, an unknown signer, a broken link, a rollback or a fork."""
|
|
80
|
+
|
|
81
|
+
code = -15
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class NodusErasurePendingError(NodusError):
|
|
85
|
+
"""The erasure is applied, but a copy of the name outside memory was not yet scrubbed when the wait ended."""
|
|
86
|
+
|
|
87
|
+
code = -16
|
|
88
|
+
token = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class NodusHistoryExpiredError(NodusError):
|
|
92
|
+
"""The past this call asks for is no longer kept: a watch cursor older than every source of changes, or a point
|
|
93
|
+
in time before what the bucket still holds. Read the current state and start again from the current token."""
|
|
94
|
+
|
|
95
|
+
code = -17
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
_BY_CODE = {
|
|
99
|
+
error.code: error
|
|
100
|
+
for error in (
|
|
101
|
+
NodusMemoryError, NodusStaleReadError, NodusCheckDepthError, NodusSchemaError, NodusLogBacklogError,
|
|
102
|
+
NodusWriterFencedError, NodusCorruptLogError, NodusUnsupportedError, NodusUpgradeRequiredError,
|
|
103
|
+
NodusIndeterminateError, NodusTokenLostError, NodusShipTimeoutError, NodusChainTrustError,
|
|
104
|
+
NodusErasurePendingError, NodusHistoryExpiredError,
|
|
105
|
+
)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def error_for(code, message):
|
|
110
|
+
return _BY_CODE.get(code, NodusError)(message)
|
nodusdb/follower.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import ctypes
|
|
2
|
+
import dataclasses
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import typing
|
|
6
|
+
|
|
7
|
+
from . import _native
|
|
8
|
+
from .errors import error_for
|
|
9
|
+
from .stores import directory_store, s3_store, without_none
|
|
10
|
+
|
|
11
|
+
Path = typing.Union[str, os.PathLike]
|
|
12
|
+
SALVAGE_CAPACITY = 1 << 16
|
|
13
|
+
SALVAGE_ATTEMPTS = 3
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclasses.dataclass(frozen=True)
|
|
17
|
+
class Follower:
|
|
18
|
+
"""Where a read-only follower reads its chain from, and how it follows it.
|
|
19
|
+
|
|
20
|
+
A follower rebuilds the graph from the newest verified snapshot in the object store and keeps applying the
|
|
21
|
+
signed chain. It holds only the public key of the writer. Every write through a follower raises
|
|
22
|
+
NodusUnsupportedError. Values are checked when the follower opens."""
|
|
23
|
+
|
|
24
|
+
store: typing.Mapping[str, typing.Any]
|
|
25
|
+
public_key_file: Path
|
|
26
|
+
key_id: int
|
|
27
|
+
credentials: typing.Optional[typing.Mapping[str, typing.Any]] = None
|
|
28
|
+
poll_interval_ms: typing.Optional[int] = None
|
|
29
|
+
read_wait_ms: typing.Optional[int] = None
|
|
30
|
+
max_staleness_ms: typing.Optional[int] = None
|
|
31
|
+
download_parallelism: typing.Optional[int] = None
|
|
32
|
+
request_timeout_ms: typing.Optional[int] = None
|
|
33
|
+
state_directory: typing.Optional[Path] = None
|
|
34
|
+
history_max_memory_mb: typing.Optional[int] = None
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def directory(cls, path, **settings):
|
|
38
|
+
"""Follow a chain in a local directory. Meant for tests and single-machine setups."""
|
|
39
|
+
return cls(store=directory_store(path), **settings)
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def s3(cls, bucket, region, *, prefix=None, endpoint=None, path_style=None, ca_bundle=None, **settings):
|
|
43
|
+
"""Follow a chain in an S3 bucket, or any S3-compatible server when endpoint is given."""
|
|
44
|
+
return cls(store=s3_store(bucket, region, prefix, endpoint, path_style, ca_bundle), **settings)
|
|
45
|
+
|
|
46
|
+
def to_json(self):
|
|
47
|
+
if isinstance(self.key_id, bool) or not isinstance(self.key_id, int):
|
|
48
|
+
raise TypeError(f"key_id must be an int, got {type(self.key_id).__name__}")
|
|
49
|
+
document = {
|
|
50
|
+
"store": dict(self.store),
|
|
51
|
+
"trust": {"key_id": self.key_id, "public_key_file": os.fspath(self.public_key_file)},
|
|
52
|
+
}
|
|
53
|
+
if self.credentials is not None:
|
|
54
|
+
document["credentials"] = dict(self.credentials)
|
|
55
|
+
follow = without_none({
|
|
56
|
+
"poll_interval_ms": self.poll_interval_ms,
|
|
57
|
+
"read_wait_ms": self.read_wait_ms,
|
|
58
|
+
"max_staleness_ms": self.max_staleness_ms,
|
|
59
|
+
"download_parallelism": self.download_parallelism,
|
|
60
|
+
"request_timeout_ms": self.request_timeout_ms,
|
|
61
|
+
"state_directory": None if self.state_directory is None else os.fspath(self.state_directory),
|
|
62
|
+
})
|
|
63
|
+
if follow:
|
|
64
|
+
document["follow"] = follow
|
|
65
|
+
if self.history_max_memory_mb is not None:
|
|
66
|
+
document["history"] = {"max_memory_mb": self.history_max_memory_mb}
|
|
67
|
+
return json.dumps(document)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclasses.dataclass(frozen=True)
|
|
71
|
+
class Restored:
|
|
72
|
+
"""What a restore wrote: the last LSN it applied, the writer epoch it ended in and the snapshot it started from."""
|
|
73
|
+
|
|
74
|
+
applied_lsn: int
|
|
75
|
+
epoch: int
|
|
76
|
+
snapshot_lsn: int
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def restore(follower, target, library_path=None):
|
|
80
|
+
"""Rebuild a graph directory from the chain a Follower describes.
|
|
81
|
+
|
|
82
|
+
The target must not exist or must be an empty directory. It appears only when the restore is complete, so a
|
|
83
|
+
failed or interrupted restore leaves no directory that looks like a graph."""
|
|
84
|
+
if not isinstance(follower, Follower):
|
|
85
|
+
raise TypeError(f"expected a Follower, got {type(follower).__name__}")
|
|
86
|
+
library, isolate = _native.load(library_path)
|
|
87
|
+
if not hasattr(library, "nodus_restore"):
|
|
88
|
+
raise error_for(-10, "this libnodusdb has no restore support; rebuild or upgrade it")
|
|
89
|
+
function = library.nodus_restore
|
|
90
|
+
function.restype = ctypes.c_int
|
|
91
|
+
function.argtypes = [_native.THREAD, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p,
|
|
92
|
+
ctypes.POINTER(ctypes.c_int64)]
|
|
93
|
+
document = follower.to_json().encode("utf-8")
|
|
94
|
+
report = (ctypes.c_int64 * 3)()
|
|
95
|
+
thread = _native.current_thread(library, isolate)
|
|
96
|
+
code = function(thread, document, len(document), os.fsencode(os.fspath(target)), report)
|
|
97
|
+
if code != 0:
|
|
98
|
+
detail = _native.last_error_message(library, thread)
|
|
99
|
+
raise error_for(code, f"restore failed: {detail}" if detail else "restore failed")
|
|
100
|
+
return Restored(applied_lsn=report[0], epoch=report[1], snapshot_lsn=report[2])
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclasses.dataclass(frozen=True)
|
|
104
|
+
class TakeoverReport:
|
|
105
|
+
"""What a takeover did: the epoch it claimed to fence the old writer, the epoch the new writer runs in, the
|
|
106
|
+
last LSN of the old writer it carries over, and how many restores it needed."""
|
|
107
|
+
|
|
108
|
+
claimed_epoch: int
|
|
109
|
+
epoch: int
|
|
110
|
+
handoff_lsn: int
|
|
111
|
+
attempts: int
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@dataclasses.dataclass(frozen=True)
|
|
115
|
+
class SalvagedChange:
|
|
116
|
+
kind: str
|
|
117
|
+
object: str
|
|
118
|
+
relation: str
|
|
119
|
+
subject: str
|
|
120
|
+
subject_relation: str
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@dataclasses.dataclass(frozen=True)
|
|
124
|
+
class SalvagedTransaction:
|
|
125
|
+
first_lsn: int
|
|
126
|
+
last_lsn: int
|
|
127
|
+
commit_micros: int
|
|
128
|
+
changes: typing.Tuple[SalvagedChange, ...]
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@dataclasses.dataclass(frozen=True)
|
|
132
|
+
class SalvageReport:
|
|
133
|
+
"""What an old writer's directory holds above the point where the chain moved on.
|
|
134
|
+
|
|
135
|
+
handoff_lsn is the last LSN the chain keeps. It is provisional when no later writer has taken over yet, and
|
|
136
|
+
then it is only the end of the chain. unavailable_through_lsn is non-zero when a checkpoint trimmed
|
|
137
|
+
transactions above the handoff from the log, so they cannot be listed."""
|
|
138
|
+
|
|
139
|
+
handoff_lsn: int
|
|
140
|
+
provisional: bool
|
|
141
|
+
local_last_lsn: int
|
|
142
|
+
unavailable_through_lsn: int
|
|
143
|
+
transactions: typing.Tuple[SalvagedTransaction, ...]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def salvage(follower, directory, library_path=None):
|
|
147
|
+
"""List the transactions in an old writer's directory that never reached the chain. Nothing is written.
|
|
148
|
+
|
|
149
|
+
The directory must not be open in any process."""
|
|
150
|
+
if not isinstance(follower, Follower):
|
|
151
|
+
raise TypeError(f"expected a Follower, got {type(follower).__name__}")
|
|
152
|
+
library, isolate = _native.load(library_path)
|
|
153
|
+
if not hasattr(library, "nodus_salvage_json"):
|
|
154
|
+
raise error_for(-10, "this libnodusdb has no salvage support; rebuild or upgrade it")
|
|
155
|
+
function = library.nodus_salvage_json
|
|
156
|
+
function.restype = ctypes.c_int
|
|
157
|
+
function.argtypes = [_native.THREAD, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
|
|
158
|
+
ctypes.c_int]
|
|
159
|
+
document = follower.to_json().encode("utf-8")
|
|
160
|
+
thread = _native.current_thread(library, isolate)
|
|
161
|
+
capacity = SALVAGE_CAPACITY
|
|
162
|
+
for _ in range(SALVAGE_ATTEMPTS):
|
|
163
|
+
buffer = ctypes.create_string_buffer(capacity)
|
|
164
|
+
length = function(thread, document, len(document), os.fsencode(os.fspath(directory)), buffer, capacity)
|
|
165
|
+
if length < 0:
|
|
166
|
+
detail = _native.last_error_message(library, thread)
|
|
167
|
+
raise error_for(length, f"salvage failed: {detail}" if detail else "salvage failed")
|
|
168
|
+
if length <= capacity:
|
|
169
|
+
return _salvage_report(json.loads(buffer.raw[:length].decode("utf-8")))
|
|
170
|
+
capacity = length
|
|
171
|
+
raise error_for(-1, "the salvage report kept growing while it was being read")
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _salvage_report(document):
|
|
175
|
+
transactions = tuple(
|
|
176
|
+
SalvagedTransaction(
|
|
177
|
+
first_lsn=item["first_lsn"], last_lsn=item["last_lsn"], commit_micros=item["commit_micros"],
|
|
178
|
+
changes=tuple(
|
|
179
|
+
SalvagedChange(kind=change["kind"], object=change["object"], relation=change["relation"],
|
|
180
|
+
subject=change["subject"], subject_relation=change["subject_relation"])
|
|
181
|
+
for change in item["changes"]))
|
|
182
|
+
for item in document["transactions"])
|
|
183
|
+
return SalvageReport(
|
|
184
|
+
handoff_lsn=document["handoff_lsn"], provisional=document["provisional"],
|
|
185
|
+
local_last_lsn=document["local_last_lsn"], unavailable_through_lsn=document["unavailable_through_lsn"],
|
|
186
|
+
transactions=transactions)
|