solera 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- solera/__init__.py +106 -0
- solera/executors.py +138 -0
- solera/ids.py +28 -0
- solera/keys/__init__.py +75 -0
- solera/keys/_python.py +532 -0
- solera/keys/index.py +790 -0
- solera/keys/io.py +230 -0
- solera/sdk.py +1247 -0
- solera/stores.py +788 -0
- solera-0.0.1.dist-info/METADATA +366 -0
- solera-0.0.1.dist-info/RECORD +52 -0
- solera-0.0.1.dist-info/WHEEL +4 -0
- solera-0.0.1.dist-info/entry_points.txt +2 -0
- solera-0.0.1.dist-info/licenses/LICENSE +15 -0
- solera_postgres/__init__.py +581 -0
- solera_server/api.py +485 -0
- solera_server/cli.py +402 -0
- solera_server/demo.py +391 -0
- solera_server/engine.py +1835 -0
- solera_server/journal.py +363 -0
- solera_server/model.py +632 -0
- solera_server/placements/__init__.py +122 -0
- solera_server/placements/inline.py +49 -0
- solera_server/placements/local.py +176 -0
- solera_server/placements/pool.py +43 -0
- solera_server/placements/remote.py +220 -0
- solera_server/selftest.py +104 -0
- solera_server/state.py +274 -0
- solera_server/web/assets/assets-BSap0vq5.js +1 -0
- solera_server/web/assets/automations-DdLiPOz0.js +1 -0
- solera_server/web/assets/button-Dq2m6pM8.js +1 -0
- solera_server/web/assets/common-B-CWyYtx.js +1 -0
- solera_server/web/assets/executors-DQQbyBAS.js +1 -0
- solera_server/web/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- solera_server/web/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- solera_server/web/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- solera_server/web/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- solera_server/web/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- solera_server/web/assets/index-JGfsGvyn.js +17 -0
- solera_server/web/assets/refresh-cw-L1RN7bT3.js +1 -0
- solera_server/web/assets/runs-CKQ9Hxux.js +1 -0
- solera_server/web/assets/sources-CFtEjXLV.js +1 -0
- solera_server/web/assets/storage-ws5IRrKT.js +1 -0
- solera_server/web/assets/styles-COw0wuCY.css +2 -0
- solera_server/web/assets/table-DwKiY1Zz.js +1 -0
- solera_server/web/assets/useSelector-o7Ls85E6.js +1 -0
- solera_server/web/assets/with-selector-CxLBpKjg.js +1 -0
- solera_server/web/favicon.svg +1 -0
- solera_server/web/index.html +0 -0
- solera_worker/__init__.py +1 -0
- solera_worker/__main__.py +5 -0
- solera_worker/worker.py +600 -0
solera/__init__.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
from .executors import AWSECS, K8sJob, Local, Modal, Placement, Pool
|
|
2
|
+
from .sdk import (
|
|
3
|
+
UNSET,
|
|
4
|
+
AllPartitions,
|
|
5
|
+
Asset,
|
|
6
|
+
Automation,
|
|
7
|
+
AutoRefresh,
|
|
8
|
+
BlobRef,
|
|
9
|
+
Changes,
|
|
10
|
+
Cron,
|
|
11
|
+
Every,
|
|
12
|
+
In,
|
|
13
|
+
Incremental,
|
|
14
|
+
JsonRef,
|
|
15
|
+
KeyCache,
|
|
16
|
+
Migration,
|
|
17
|
+
OnChange,
|
|
18
|
+
OnDeploy,
|
|
19
|
+
Output,
|
|
20
|
+
PartitionSet,
|
|
21
|
+
Project,
|
|
22
|
+
Ref,
|
|
23
|
+
RegistrationError,
|
|
24
|
+
Result,
|
|
25
|
+
Retention,
|
|
26
|
+
Retry,
|
|
27
|
+
Source,
|
|
28
|
+
StaticPartitions,
|
|
29
|
+
TableRef,
|
|
30
|
+
TimePartitions,
|
|
31
|
+
asset,
|
|
32
|
+
canonical_partition,
|
|
33
|
+
digest,
|
|
34
|
+
job,
|
|
35
|
+
split_partition,
|
|
36
|
+
)
|
|
37
|
+
from .stores import (
|
|
38
|
+
Batches,
|
|
39
|
+
BlobStore,
|
|
40
|
+
JsonStore,
|
|
41
|
+
Keys,
|
|
42
|
+
Patch,
|
|
43
|
+
Scope,
|
|
44
|
+
Sql,
|
|
45
|
+
StaleRead,
|
|
46
|
+
Store,
|
|
47
|
+
StoreConflict,
|
|
48
|
+
StoreError,
|
|
49
|
+
WriteError,
|
|
50
|
+
Written,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
__all__ = [
|
|
54
|
+
"AWSECS",
|
|
55
|
+
"AllPartitions",
|
|
56
|
+
"Asset",
|
|
57
|
+
"Automation",
|
|
58
|
+
"AutoRefresh",
|
|
59
|
+
"Batches",
|
|
60
|
+
"BlobRef",
|
|
61
|
+
"BlobStore",
|
|
62
|
+
"Changes",
|
|
63
|
+
"Cron",
|
|
64
|
+
"Every",
|
|
65
|
+
"In",
|
|
66
|
+
"Incremental",
|
|
67
|
+
"JsonRef",
|
|
68
|
+
"JsonStore",
|
|
69
|
+
"K8sJob",
|
|
70
|
+
"KeyCache",
|
|
71
|
+
"Keys",
|
|
72
|
+
"Local",
|
|
73
|
+
"Migration",
|
|
74
|
+
"Modal",
|
|
75
|
+
"OnChange",
|
|
76
|
+
"OnDeploy",
|
|
77
|
+
"Output",
|
|
78
|
+
"PartitionSet",
|
|
79
|
+
"Patch",
|
|
80
|
+
"Placement",
|
|
81
|
+
"Pool",
|
|
82
|
+
"Project",
|
|
83
|
+
"Ref",
|
|
84
|
+
"RegistrationError",
|
|
85
|
+
"Result",
|
|
86
|
+
"Retention",
|
|
87
|
+
"Retry",
|
|
88
|
+
"Scope",
|
|
89
|
+
"Source",
|
|
90
|
+
"Sql",
|
|
91
|
+
"StaleRead",
|
|
92
|
+
"StaticPartitions",
|
|
93
|
+
"Store",
|
|
94
|
+
"StoreConflict",
|
|
95
|
+
"StoreError",
|
|
96
|
+
"TableRef",
|
|
97
|
+
"TimePartitions",
|
|
98
|
+
"UNSET",
|
|
99
|
+
"WriteError",
|
|
100
|
+
"Written",
|
|
101
|
+
"asset",
|
|
102
|
+
"canonical_partition",
|
|
103
|
+
"digest",
|
|
104
|
+
"job",
|
|
105
|
+
"split_partition",
|
|
106
|
+
]
|
solera/executors.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Environments and placements (§10). Calling an environment returns a placement."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from .sdk import RegistrationError
|
|
10
|
+
|
|
11
|
+
_BYTES = {
|
|
12
|
+
"B": 1,
|
|
13
|
+
"KB": 10**3,
|
|
14
|
+
"MB": 10**6,
|
|
15
|
+
"GB": 10**9,
|
|
16
|
+
"TB": 10**12,
|
|
17
|
+
"KIB": 2**10,
|
|
18
|
+
"MIB": 2**20,
|
|
19
|
+
"GIB": 2**30,
|
|
20
|
+
"TIB": 2**40,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def parse_bytes(value: str | int | None) -> int | None:
|
|
25
|
+
if value is None or isinstance(value, int):
|
|
26
|
+
return value
|
|
27
|
+
match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([A-Za-z]+)", str(value).strip())
|
|
28
|
+
if not match or match.group(2).upper() not in _BYTES:
|
|
29
|
+
raise RegistrationError(f"Invalid memory value: {value!r}")
|
|
30
|
+
return int(float(match.group(1)) * _BYTES[match.group(2).upper()])
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class Placement:
|
|
35
|
+
"""A typed per-asset request built from an environment (§10)."""
|
|
36
|
+
|
|
37
|
+
kind: str
|
|
38
|
+
environment: dict
|
|
39
|
+
options: dict
|
|
40
|
+
|
|
41
|
+
def serialized(self) -> dict:
|
|
42
|
+
return {"kind": self.kind, "environment": self.environment, "placement": self.options}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Environment:
|
|
46
|
+
kind = "?"
|
|
47
|
+
allowed: frozenset[str] = frozenset()
|
|
48
|
+
|
|
49
|
+
def __init__(self, **config: Any):
|
|
50
|
+
self.config = config
|
|
51
|
+
|
|
52
|
+
def _options(self, **options: Any) -> dict:
|
|
53
|
+
return {k: v for k, v in options.items() if v is not None}
|
|
54
|
+
|
|
55
|
+
def _check(self, options: dict, allowed: set[str]) -> dict:
|
|
56
|
+
unknown = set(options) - allowed
|
|
57
|
+
if unknown:
|
|
58
|
+
raise RegistrationError(f"{self.kind} does not take options: {sorted(unknown)}")
|
|
59
|
+
if "cpu" in options and options["cpu"] is not None and not isinstance(options["cpu"], int):
|
|
60
|
+
raise RegistrationError(f"{self.kind}: cpu must be an int")
|
|
61
|
+
if "memory" in options:
|
|
62
|
+
options["memory"] = parse_bytes(options["memory"])
|
|
63
|
+
return {k: v for k, v in options.items() if v is not None}
|
|
64
|
+
|
|
65
|
+
def __call__(self, **options) -> Placement:
|
|
66
|
+
return Placement(self.kind, dict(self.config), self._check(options, set(self.allowed)))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class Local(Environment):
|
|
70
|
+
"""In-process/subprocess on the engine host. No options."""
|
|
71
|
+
|
|
72
|
+
kind = "Local"
|
|
73
|
+
|
|
74
|
+
def __init__(self):
|
|
75
|
+
super().__init__()
|
|
76
|
+
|
|
77
|
+
def __call__(self, **options) -> Placement:
|
|
78
|
+
return Placement(self.kind, dict(self.config), self._check(options, set()))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class AWSECS(Environment):
|
|
82
|
+
kind = "AWSECS"
|
|
83
|
+
|
|
84
|
+
def __init__(self, *, cluster: str, region: str):
|
|
85
|
+
super().__init__(cluster=cluster, region=region)
|
|
86
|
+
|
|
87
|
+
def __call__(self, *, cpu: int | None = None, memory=None, gpu=None, image=None) -> Placement:
|
|
88
|
+
return Placement(
|
|
89
|
+
self.kind,
|
|
90
|
+
dict(self.config),
|
|
91
|
+
self._check(
|
|
92
|
+
{"cpu": cpu, "memory": memory, "gpu": gpu, "image": image},
|
|
93
|
+
{"cpu", "memory", "gpu", "image"},
|
|
94
|
+
),
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class K8sJob(Environment):
|
|
99
|
+
kind = "K8sJob"
|
|
100
|
+
|
|
101
|
+
def __init__(self, *, cluster: str, namespace: str = "default"):
|
|
102
|
+
super().__init__(cluster=cluster, namespace=namespace)
|
|
103
|
+
|
|
104
|
+
def __call__(self, *, cpu: int | None = None, memory=None, image=None) -> Placement:
|
|
105
|
+
return Placement(
|
|
106
|
+
self.kind,
|
|
107
|
+
dict(self.config),
|
|
108
|
+
self._check({"cpu": cpu, "memory": memory, "image": image}, {"cpu", "memory", "image"}),
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class Modal(Environment):
|
|
113
|
+
kind = "Modal"
|
|
114
|
+
|
|
115
|
+
def __init__(self, *, app: str):
|
|
116
|
+
super().__init__(app=app)
|
|
117
|
+
|
|
118
|
+
def __call__(self, *, gpu=None) -> Placement:
|
|
119
|
+
return Placement(self.kind, dict(self.config), self._check({"gpu": gpu}, {"gpu"}))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class Pool(Environment):
|
|
123
|
+
"""Pull path: external workers claim stages through the API (§10)."""
|
|
124
|
+
|
|
125
|
+
kind = "Pool"
|
|
126
|
+
|
|
127
|
+
def __init__(self, name: str):
|
|
128
|
+
super().__init__(name=name)
|
|
129
|
+
|
|
130
|
+
def __call__(self, *, cpu: int | None = None, memory=None, gpu=None) -> Placement:
|
|
131
|
+
return Placement(
|
|
132
|
+
self.kind,
|
|
133
|
+
dict(self.config),
|
|
134
|
+
self._check({"cpu": cpu, "memory": memory, "gpu": gpu}, {"cpu", "memory", "gpu"}),
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
BUILTIN_KINDS = {k.kind: k for k in (Local, AWSECS, K8sJob, Modal, Pool)}
|
solera/ids.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Time-sortable unique ids (ULID): 48-bit millisecond timestamp + 80 random bits,
|
|
2
|
+
Crockford base32, 26 characters. Lexicographic order is creation order."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import os
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
_ALPHABET = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def ulid(now: float | None = None) -> str:
|
|
13
|
+
ms = int((time.time() if now is None else now) * 1000) & ((1 << 48) - 1)
|
|
14
|
+
n = (ms << 80) | int.from_bytes(os.urandom(10), "big")
|
|
15
|
+
out = []
|
|
16
|
+
for _ in range(26):
|
|
17
|
+
out.append(_ALPHABET[n & 31])
|
|
18
|
+
n >>= 5
|
|
19
|
+
return "".join(reversed(out))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def ulid_time(value: str) -> float:
|
|
23
|
+
"""The creation time (seconds) embedded in a ULID."""
|
|
24
|
+
|
|
25
|
+
n = 0
|
|
26
|
+
for ch in value[:10]:
|
|
27
|
+
n = (n << 5) | _ALPHABET.index(ch)
|
|
28
|
+
return n / 1000
|
solera/keys/__init__.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""The engine-owned key index (docs/object-store-state.md §6).
|
|
2
|
+
|
|
3
|
+
`.kx` files are encoded and decoded by the `solera_native` extension when it
|
|
4
|
+
is installed, else by the pure-Python reference in `_python`; both follow
|
|
5
|
+
docs/key-index-format.md. Set `SOLERA_KEYS_IMPL=python` to force the
|
|
6
|
+
reference. Tails are always parsed in Python (they are small); the
|
|
7
|
+
per-entry work — encoding, decoding, filter checks, sorting, merging — is
|
|
8
|
+
what the native extension accelerates.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
|
|
15
|
+
from . import _python
|
|
16
|
+
from ._python import (
|
|
17
|
+
CODEC_NONE,
|
|
18
|
+
CODEC_ZLIB,
|
|
19
|
+
FOOTER_SIZE,
|
|
20
|
+
FormatError,
|
|
21
|
+
check_block,
|
|
22
|
+
filter_nbits,
|
|
23
|
+
parse_footer,
|
|
24
|
+
parse_index,
|
|
25
|
+
parse_tail,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load_native():
|
|
30
|
+
if os.environ.get("SOLERA_KEYS_IMPL", "").lower() == "python":
|
|
31
|
+
return None
|
|
32
|
+
try:
|
|
33
|
+
import solera_native
|
|
34
|
+
except ImportError:
|
|
35
|
+
return None
|
|
36
|
+
return solera_native
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
_native = _load_native()
|
|
40
|
+
IMPL = "native" if _native is not None else "python"
|
|
41
|
+
_impl = _native or _python
|
|
42
|
+
|
|
43
|
+
encode_file = _impl.encode_file
|
|
44
|
+
decode_block = _impl.decode_block
|
|
45
|
+
bloom_check_keys = _impl.bloom_check_keys
|
|
46
|
+
bloom_check_pairs = _impl.bloom_check_pairs
|
|
47
|
+
bloom_check_tombstones = _impl.bloom_check_tombstones
|
|
48
|
+
sort_entries = _impl.sort_entries
|
|
49
|
+
merge_files = _impl.merge_files
|
|
50
|
+
lookup = _impl.lookup
|
|
51
|
+
merge_range = _impl.merge_range
|
|
52
|
+
replace_diff = _impl.replace_diff
|
|
53
|
+
|
|
54
|
+
__all__ = [
|
|
55
|
+
"CODEC_NONE",
|
|
56
|
+
"CODEC_ZLIB",
|
|
57
|
+
"FOOTER_SIZE",
|
|
58
|
+
"IMPL",
|
|
59
|
+
"FormatError",
|
|
60
|
+
"bloom_check_keys",
|
|
61
|
+
"bloom_check_pairs",
|
|
62
|
+
"bloom_check_tombstones",
|
|
63
|
+
"check_block",
|
|
64
|
+
"decode_block",
|
|
65
|
+
"encode_file",
|
|
66
|
+
"filter_nbits",
|
|
67
|
+
"lookup",
|
|
68
|
+
"merge_files",
|
|
69
|
+
"merge_range",
|
|
70
|
+
"parse_footer",
|
|
71
|
+
"parse_index",
|
|
72
|
+
"parse_tail",
|
|
73
|
+
"replace_diff",
|
|
74
|
+
"sort_entries",
|
|
75
|
+
]
|