nodusdb 0.1.0__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
nodusdb/__init__.py ADDED
@@ -0,0 +1,88 @@
1
+ from ._native import find_library
2
+ from ._version import __version__
3
+ from .errors import (
4
+ NodusChainTrustError,
5
+ NodusCheckDepthError,
6
+ NodusCorruptLogError,
7
+ NodusError,
8
+ NodusErasurePendingError,
9
+ NodusHistoryExpiredError,
10
+ NodusIndeterminateError,
11
+ NodusLogBacklogError,
12
+ NodusMemoryError,
13
+ NodusSchemaError,
14
+ NodusShipTimeoutError,
15
+ NodusStaleReadError,
16
+ NodusTokenLostError,
17
+ NodusUnsupportedError,
18
+ NodusUpgradeRequiredError,
19
+ NodusWriterFencedError,
20
+ )
21
+ from .follower import (
22
+ Follower,
23
+ Restored,
24
+ SalvagedChange,
25
+ SalvagedTransaction,
26
+ SalvageReport,
27
+ TakeoverReport,
28
+ restore,
29
+ salvage,
30
+ )
31
+ from .graph import Graph, PastGraph, Token, Transaction
32
+ from .lake import LakeTable, key_hash
33
+ from .shipping import (
34
+ Shipping,
35
+ environment_credentials,
36
+ file_credentials,
37
+ generate_signing_key,
38
+ static_credentials,
39
+ )
40
+ from .upgrade import UpgradeReport, upgrade, upgrade_cleanup
41
+ from .watch import AccessChange, AccessWatch, Change, ChangeBatch
42
+
43
+ __all__ = [
44
+ "__version__",
45
+ "AccessChange",
46
+ "AccessWatch",
47
+ "Change",
48
+ "ChangeBatch",
49
+ "Follower",
50
+ "Graph",
51
+ "LakeTable",
52
+ "NodusChainTrustError",
53
+ "NodusCheckDepthError",
54
+ "NodusCorruptLogError",
55
+ "NodusError",
56
+ "NodusErasurePendingError",
57
+ "NodusHistoryExpiredError",
58
+ "NodusIndeterminateError",
59
+ "NodusLogBacklogError",
60
+ "NodusMemoryError",
61
+ "NodusSchemaError",
62
+ "NodusShipTimeoutError",
63
+ "NodusStaleReadError",
64
+ "NodusTokenLostError",
65
+ "NodusUnsupportedError",
66
+ "NodusUpgradeRequiredError",
67
+ "NodusWriterFencedError",
68
+ "PastGraph",
69
+ "Restored",
70
+ "SalvageReport",
71
+ "SalvagedChange",
72
+ "SalvagedTransaction",
73
+ "Shipping",
74
+ "TakeoverReport",
75
+ "Token",
76
+ "Transaction",
77
+ "UpgradeReport",
78
+ "environment_credentials",
79
+ "file_credentials",
80
+ "find_library",
81
+ "generate_signing_key",
82
+ "key_hash",
83
+ "restore",
84
+ "salvage",
85
+ "static_credentials",
86
+ "upgrade",
87
+ "upgrade_cleanup",
88
+ ]
nodusdb/_native.py ADDED
@@ -0,0 +1,162 @@
1
+ import ctypes
2
+ import functools
3
+ import os
4
+ import platform
5
+ import sys
6
+ import threading
7
+ import warnings
8
+ from pathlib import Path
9
+
10
+ from .errors import NodusError, NodusMemoryError, error_for # noqa: F401
11
+
12
+ ENVIRONMENT_VARIABLE = "NODUSDB_LIBRARY"
13
+
14
+ ERROR = -1
15
+ MEMORY_LIMIT = -3
16
+
17
+ PACKAGE_DIRECTORY = Path(__file__).resolve().parent
18
+ REPOSITORY_ROOT = PACKAGE_DIRECTORY.parents[1]
19
+
20
+ HANDLE = ctypes.c_void_p
21
+ THREAD = ctypes.c_void_p
22
+ NODE = ctypes.c_int64
23
+ RESULT_BUFFER = ctypes.POINTER(ctypes.c_int64)
24
+
25
+ _OPERATING_SYSTEMS = {"Windows": "windows", "Darwin": "macos", "Linux": "linux"}
26
+ _ARCHITECTURES = {"amd64": "x86_64", "x86_64": "x86_64", "arm64": "arm64", "aarch64": "arm64"}
27
+
28
+ _loaded = {}
29
+ _attached = threading.local()
30
+
31
+
32
+ def platform_tag():
33
+ system = platform.system()
34
+ operating_system = _OPERATING_SYSTEMS.get(system, system.lower())
35
+ machine = platform.machine().lower()
36
+ architecture = _ARCHITECTURES.get(machine, machine)
37
+ return f"{operating_system}-{architecture}"
38
+
39
+
40
+ def _library_file_names():
41
+ if sys.platform == "win32":
42
+ return ("libnodusdb.dll", "nodusdb.dll")
43
+ if sys.platform == "darwin":
44
+ return ("libnodusdb.dylib",)
45
+ return ("libnodusdb.so",)
46
+
47
+
48
+ def _search_directories():
49
+ return (
50
+ PACKAGE_DIRECTORY / "bin" / platform_tag(),
51
+ REPOSITORY_ROOT / "target" / "native",
52
+ )
53
+
54
+
55
+ def find_library(explicit_path=None):
56
+ if explicit_path is not None:
57
+ candidate = Path(explicit_path)
58
+ if candidate.is_file():
59
+ return candidate
60
+ raise NodusError(f"native library not found: {candidate}")
61
+
62
+ from_environment = os.environ.get(ENVIRONMENT_VARIABLE)
63
+ if from_environment:
64
+ return find_library(from_environment)
65
+
66
+ for directory in _search_directories():
67
+ for name in _library_file_names():
68
+ candidate = directory / name
69
+ if candidate.is_file():
70
+ return candidate
71
+
72
+ raise NodusError(
73
+ "libnodusdb not found; build it with `mvn -Pnative -pl core package` "
74
+ f"or set {ENVIRONMENT_VARIABLE} to the library path"
75
+ )
76
+
77
+
78
+ def _create_isolate(library):
79
+ create = library.graal_create_isolate
80
+ create.restype = ctypes.c_int
81
+ create.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_void_p)]
82
+ isolate = ctypes.c_void_p()
83
+ thread = ctypes.c_void_p()
84
+ if create(None, ctypes.byref(isolate), ctypes.byref(thread)) != 0 or not isolate.value:
85
+ raise NodusError("failed to create the GraalVM isolate")
86
+ return isolate
87
+
88
+
89
+ def serialized(method):
90
+ @functools.wraps(method)
91
+ def locked(self, *args, **kwargs):
92
+ with self._lock:
93
+ return method(self, *args, **kwargs)
94
+ return locked
95
+
96
+
97
+ def current_thread(library, isolate):
98
+ """Return this OS thread's isolate thread, attaching it on first use."""
99
+ attached = _attached.__dict__.setdefault("threads", {})
100
+ thread = attached.get(isolate.value)
101
+ if thread is None:
102
+ attach = library.graal_attach_thread
103
+ attach.restype = ctypes.c_int
104
+ attach.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
105
+ candidate = ctypes.c_void_p()
106
+ if attach(isolate, ctypes.byref(candidate)) != 0 or not candidate.value:
107
+ raise NodusError("failed to attach this thread to the GraalVM isolate")
108
+ thread = candidate
109
+ attached[isolate.value] = thread
110
+ return thread
111
+
112
+
113
+ def last_error_message(library, thread):
114
+ """The message of the failure this thread's last native call reported."""
115
+ function = library.nodus_last_error
116
+ function.restype = ctypes.c_int
117
+ function.argtypes = [THREAD, ctypes.c_char_p, ctypes.c_int]
118
+ buffer = ctypes.create_string_buffer(512)
119
+ length = function(thread, buffer, len(buffer))
120
+ if length > len(buffer):
121
+ buffer = ctypes.create_string_buffer(length)
122
+ length = function(thread, buffer, len(buffer))
123
+ return buffer.raw[:max(length, 0)].decode("utf-8", "replace")
124
+
125
+
126
+ class OwnedHandle:
127
+ """A native handle and the call that releases it, shared by its owner and the owner's finalizer.
128
+
129
+ It must not refer to the owner, or the finalizer would keep the owner alive."""
130
+
131
+ def __init__(self, library, isolate, value, release, failure):
132
+ self._library = library
133
+ self._isolate = isolate
134
+ self._release = release
135
+ self._failure = failure
136
+ self.value = value
137
+
138
+ def close(self):
139
+ if not self.value:
140
+ return
141
+ if not self._release(current_thread(self._library, self._isolate), self.value):
142
+ raise NodusError(self._failure)
143
+ self.value = None
144
+
145
+
146
+ def release_unclosed(owned, kind):
147
+ """Finalizer body: free a handle whose owner was collected without close()."""
148
+ if not owned.value:
149
+ return
150
+ warnings.warn(f"unclosed nodusdb {kind}; call close() or use a with block", ResourceWarning, stacklevel=2)
151
+ try:
152
+ owned.close()
153
+ except NodusError as error:
154
+ warnings.warn(f"nodusdb {kind} could not be closed: {error}", ResourceWarning, stacklevel=2)
155
+
156
+
157
+ def load(explicit_path=None):
158
+ path = str(find_library(explicit_path))
159
+ if path not in _loaded:
160
+ library = ctypes.CDLL(path)
161
+ _loaded[path] = (library, _create_isolate(library))
162
+ return _loaded[path]
nodusdb/_tokens.py ADDED
@@ -0,0 +1,8 @@
1
+ import typing
2
+
3
+
4
+ class Token(typing.NamedTuple):
5
+ """The position of a write: the epoch of the writer tenure and the log sequence number."""
6
+
7
+ epoch: int
8
+ lsn: int
nodusdb/_version.py ADDED
@@ -0,0 +1,6 @@
1
+ from importlib.metadata import PackageNotFoundError, version
2
+
3
+ try:
4
+ __version__ = version("nodusdb")
5
+ except PackageNotFoundError:
6
+ __version__ = "0+unknown"
@@ -0,0 +1,4 @@
1
+ [diffend] Oversized file quarantined before diffing.
2
+ name: nodusdb/bin/windows-x86_64/libnodusdb.dll
3
+ size: 39092224 bytes
4
+ sha256: a9c6f4de3428e51a90692407acaffbf86b158e84457ff22acdc9785118f7d6fe
nodusdb/errors.py ADDED
@@ -0,0 +1,110 @@
1
+ class NodusError(RuntimeError):
2
+ code = -1
3
+
4
+
5
+ class NodusMemoryError(NodusError, MemoryError):
6
+ """The graph's native memory limit would be exceeded. Existing data is unchanged."""
7
+
8
+ code = -3
9
+
10
+
11
+ class NodusStaleReadError(NodusError):
12
+ """The graph has not yet applied the write the token names."""
13
+
14
+ code = -4
15
+
16
+
17
+ class NodusCheckDepthError(NodusError):
18
+ """A check reached its depth limit before it could decide."""
19
+
20
+ code = -5
21
+
22
+
23
+ class NodusSchemaError(NodusError):
24
+ """The schema or a tuple violates the schema. Nothing changed."""
25
+
26
+ code = -6
27
+
28
+
29
+ class NodusLogBacklogError(NodusError):
30
+ """The unshipped log reached its cap."""
31
+
32
+ code = -7
33
+
34
+
35
+ class NodusWriterFencedError(NodusError):
36
+ """A newer writer exists for this graph."""
37
+
38
+ code = -8
39
+
40
+
41
+ class NodusCorruptLogError(NodusError):
42
+ """The log is damaged inside the part known to be durable."""
43
+
44
+ code = -9
45
+
46
+
47
+ class NodusUnsupportedError(NodusError):
48
+ """The operation is not available in this mode or version."""
49
+
50
+ code = -10
51
+
52
+
53
+ class NodusUpgradeRequiredError(NodusError):
54
+ """The directory was written by an earlier version; run upgrade() first."""
55
+
56
+ code = -11
57
+
58
+
59
+ class NodusIndeterminateError(NodusError):
60
+ """A log force failed, so the write may or may not have survived; reopen the graph to find out."""
61
+
62
+ code = -12
63
+
64
+
65
+ class NodusTokenLostError(NodusError):
66
+ """The write the token names was acknowledged but never became durable."""
67
+
68
+ code = -13
69
+
70
+
71
+ class NodusShipTimeoutError(NodusError):
72
+ """The write is applied and locally durable but had not reached the object store when the wait ended."""
73
+
74
+ code = -14
75
+ token = None
76
+
77
+
78
+ class NodusChainTrustError(NodusError):
79
+ """The object store holds chain data that fails verification: a bad signature, an unknown signer, a broken link, a rollback or a fork."""
80
+
81
+ code = -15
82
+
83
+
84
+ class NodusErasurePendingError(NodusError):
85
+ """The erasure is applied, but a copy of the name outside memory was not yet scrubbed when the wait ended."""
86
+
87
+ code = -16
88
+ token = None
89
+
90
+
91
+ class NodusHistoryExpiredError(NodusError):
92
+ """The past this call asks for is no longer kept: a watch cursor older than every source of changes, or a point
93
+ in time before what the bucket still holds. Read the current state and start again from the current token."""
94
+
95
+ code = -17
96
+
97
+
98
+ _BY_CODE = {
99
+ error.code: error
100
+ for error in (
101
+ NodusMemoryError, NodusStaleReadError, NodusCheckDepthError, NodusSchemaError, NodusLogBacklogError,
102
+ NodusWriterFencedError, NodusCorruptLogError, NodusUnsupportedError, NodusUpgradeRequiredError,
103
+ NodusIndeterminateError, NodusTokenLostError, NodusShipTimeoutError, NodusChainTrustError,
104
+ NodusErasurePendingError, NodusHistoryExpiredError,
105
+ )
106
+ }
107
+
108
+
109
+ def error_for(code, message):
110
+ return _BY_CODE.get(code, NodusError)(message)
nodusdb/follower.py ADDED
@@ -0,0 +1,186 @@
1
+ import ctypes
2
+ import dataclasses
3
+ import json
4
+ import os
5
+ import typing
6
+
7
+ from . import _native
8
+ from .errors import error_for
9
+ from .stores import directory_store, s3_store, without_none
10
+
11
+ Path = typing.Union[str, os.PathLike]
12
+ SALVAGE_CAPACITY = 1 << 16
13
+ SALVAGE_ATTEMPTS = 3
14
+
15
+
16
+ @dataclasses.dataclass(frozen=True)
17
+ class Follower:
18
+ """Where a read-only follower reads its chain from, and how it follows it.
19
+
20
+ A follower rebuilds the graph from the newest verified snapshot in the object store and keeps applying the
21
+ signed chain. It holds only the public key of the writer. Every write through a follower raises
22
+ NodusUnsupportedError. Values are checked when the follower opens."""
23
+
24
+ store: typing.Mapping[str, typing.Any]
25
+ public_key_file: Path
26
+ key_id: int
27
+ credentials: typing.Optional[typing.Mapping[str, typing.Any]] = None
28
+ poll_interval_ms: typing.Optional[int] = None
29
+ read_wait_ms: typing.Optional[int] = None
30
+ max_staleness_ms: typing.Optional[int] = None
31
+ download_parallelism: typing.Optional[int] = None
32
+ request_timeout_ms: typing.Optional[int] = None
33
+ state_directory: typing.Optional[Path] = None
34
+ history_max_memory_mb: typing.Optional[int] = None
35
+
36
+ @classmethod
37
+ def directory(cls, path, **settings):
38
+ """Follow a chain in a local directory. Meant for tests and single-machine setups."""
39
+ return cls(store=directory_store(path), **settings)
40
+
41
+ @classmethod
42
+ def s3(cls, bucket, region, *, prefix=None, endpoint=None, path_style=None, ca_bundle=None, **settings):
43
+ """Follow a chain in an S3 bucket, or any S3-compatible server when endpoint is given."""
44
+ return cls(store=s3_store(bucket, region, prefix, endpoint, path_style, ca_bundle), **settings)
45
+
46
+ def to_json(self):
47
+ if isinstance(self.key_id, bool) or not isinstance(self.key_id, int):
48
+ raise TypeError(f"key_id must be an int, got {type(self.key_id).__name__}")
49
+ document = {
50
+ "store": dict(self.store),
51
+ "trust": {"key_id": self.key_id, "public_key_file": os.fspath(self.public_key_file)},
52
+ }
53
+ if self.credentials is not None:
54
+ document["credentials"] = dict(self.credentials)
55
+ follow = without_none({
56
+ "poll_interval_ms": self.poll_interval_ms,
57
+ "read_wait_ms": self.read_wait_ms,
58
+ "max_staleness_ms": self.max_staleness_ms,
59
+ "download_parallelism": self.download_parallelism,
60
+ "request_timeout_ms": self.request_timeout_ms,
61
+ "state_directory": None if self.state_directory is None else os.fspath(self.state_directory),
62
+ })
63
+ if follow:
64
+ document["follow"] = follow
65
+ if self.history_max_memory_mb is not None:
66
+ document["history"] = {"max_memory_mb": self.history_max_memory_mb}
67
+ return json.dumps(document)
68
+
69
+
70
+ @dataclasses.dataclass(frozen=True)
71
+ class Restored:
72
+ """What a restore wrote: the last LSN it applied, the writer epoch it ended in and the snapshot it started from."""
73
+
74
+ applied_lsn: int
75
+ epoch: int
76
+ snapshot_lsn: int
77
+
78
+
79
+ def restore(follower, target, library_path=None):
80
+ """Rebuild a graph directory from the chain a Follower describes.
81
+
82
+ The target must not exist or must be an empty directory. It appears only when the restore is complete, so a
83
+ failed or interrupted restore leaves no directory that looks like a graph."""
84
+ if not isinstance(follower, Follower):
85
+ raise TypeError(f"expected a Follower, got {type(follower).__name__}")
86
+ library, isolate = _native.load(library_path)
87
+ if not hasattr(library, "nodus_restore"):
88
+ raise error_for(-10, "this libnodusdb has no restore support; rebuild or upgrade it")
89
+ function = library.nodus_restore
90
+ function.restype = ctypes.c_int
91
+ function.argtypes = [_native.THREAD, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p,
92
+ ctypes.POINTER(ctypes.c_int64)]
93
+ document = follower.to_json().encode("utf-8")
94
+ report = (ctypes.c_int64 * 3)()
95
+ thread = _native.current_thread(library, isolate)
96
+ code = function(thread, document, len(document), os.fsencode(os.fspath(target)), report)
97
+ if code != 0:
98
+ detail = _native.last_error_message(library, thread)
99
+ raise error_for(code, f"restore failed: {detail}" if detail else "restore failed")
100
+ return Restored(applied_lsn=report[0], epoch=report[1], snapshot_lsn=report[2])
101
+
102
+
103
+ @dataclasses.dataclass(frozen=True)
104
+ class TakeoverReport:
105
+ """What a takeover did: the epoch it claimed to fence the old writer, the epoch the new writer runs in, the
106
+ last LSN of the old writer it carries over, and how many restores it needed."""
107
+
108
+ claimed_epoch: int
109
+ epoch: int
110
+ handoff_lsn: int
111
+ attempts: int
112
+
113
+
114
+ @dataclasses.dataclass(frozen=True)
115
+ class SalvagedChange:
116
+ kind: str
117
+ object: str
118
+ relation: str
119
+ subject: str
120
+ subject_relation: str
121
+
122
+
123
+ @dataclasses.dataclass(frozen=True)
124
+ class SalvagedTransaction:
125
+ first_lsn: int
126
+ last_lsn: int
127
+ commit_micros: int
128
+ changes: typing.Tuple[SalvagedChange, ...]
129
+
130
+
131
+ @dataclasses.dataclass(frozen=True)
132
+ class SalvageReport:
133
+ """What an old writer's directory holds above the point where the chain moved on.
134
+
135
+ handoff_lsn is the last LSN the chain keeps. It is provisional when no later writer has taken over yet, and
136
+ then it is only the end of the chain. unavailable_through_lsn is non-zero when a checkpoint trimmed
137
+ transactions above the handoff from the log, so they cannot be listed."""
138
+
139
+ handoff_lsn: int
140
+ provisional: bool
141
+ local_last_lsn: int
142
+ unavailable_through_lsn: int
143
+ transactions: typing.Tuple[SalvagedTransaction, ...]
144
+
145
+
146
+ def salvage(follower, directory, library_path=None):
147
+ """List the transactions in an old writer's directory that never reached the chain. Nothing is written.
148
+
149
+ The directory must not be open in any process."""
150
+ if not isinstance(follower, Follower):
151
+ raise TypeError(f"expected a Follower, got {type(follower).__name__}")
152
+ library, isolate = _native.load(library_path)
153
+ if not hasattr(library, "nodus_salvage_json"):
154
+ raise error_for(-10, "this libnodusdb has no salvage support; rebuild or upgrade it")
155
+ function = library.nodus_salvage_json
156
+ function.restype = ctypes.c_int
157
+ function.argtypes = [_native.THREAD, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p,
158
+ ctypes.c_int]
159
+ document = follower.to_json().encode("utf-8")
160
+ thread = _native.current_thread(library, isolate)
161
+ capacity = SALVAGE_CAPACITY
162
+ for _ in range(SALVAGE_ATTEMPTS):
163
+ buffer = ctypes.create_string_buffer(capacity)
164
+ length = function(thread, document, len(document), os.fsencode(os.fspath(directory)), buffer, capacity)
165
+ if length < 0:
166
+ detail = _native.last_error_message(library, thread)
167
+ raise error_for(length, f"salvage failed: {detail}" if detail else "salvage failed")
168
+ if length <= capacity:
169
+ return _salvage_report(json.loads(buffer.raw[:length].decode("utf-8")))
170
+ capacity = length
171
+ raise error_for(-1, "the salvage report kept growing while it was being read")
172
+
173
+
174
+ def _salvage_report(document):
175
+ transactions = tuple(
176
+ SalvagedTransaction(
177
+ first_lsn=item["first_lsn"], last_lsn=item["last_lsn"], commit_micros=item["commit_micros"],
178
+ changes=tuple(
179
+ SalvagedChange(kind=change["kind"], object=change["object"], relation=change["relation"],
180
+ subject=change["subject"], subject_relation=change["subject_relation"])
181
+ for change in item["changes"]))
182
+ for item in document["transactions"])
183
+ return SalvageReport(
184
+ handoff_lsn=document["handoff_lsn"], provisional=document["provisional"],
185
+ local_last_lsn=document["local_last_lsn"], unavailable_through_lsn=document["unavailable_through_lsn"],
186
+ transactions=transactions)