cryptolabs-proxy 1.3.1__tar.gz → 1.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cryptolabs_proxy-1.3.1/src/cryptolabs_proxy.egg-info → cryptolabs_proxy-1.4.0}/PKG-INFO +5 -1
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/README.md +4 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/pyproject.toml +1 -1
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/__init__.py +1 -1
- cryptolabs_proxy-1.4.0/src/cryptolabs_proxy/controller_recovery.py +284 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/standard_adoption.py +147 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/updates.py +19 -6
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0/src/cryptolabs_proxy.egg-info}/PKG-INFO +5 -1
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy.egg-info/SOURCES.txt +3 -0
- cryptolabs_proxy-1.4.0/tests/test_controller_guard_transport.py +126 -0
- cryptolabs_proxy-1.4.0/tests/test_controller_recovery_guard.py +342 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_proxy_customer_route_repairs.py +7 -2
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_proxy_recovery_tls_repairs.py +1 -1
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_standard_adoption.py +2 -1
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_vast_price_manager.py +6 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/LICENSE +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/setup.cfg +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/auth.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/cli.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/config.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/custom_config.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/exporter_manager.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/identity.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/migration.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/release_selection.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/services.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/setup.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/ssl.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/acme-challenge.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/docker-compose.yml.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/exporter-auth-routes.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/fleet-api-routes.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/host-forwarding.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/legacy-bootstrap-nginx.conf +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/legacy-bootstrap-provenance.json +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/nginx.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/standard-service-routes.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/vast-price-manager-auth-blocks.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/templates/vast-price-manager.conf.j2 +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy/vpm_prerequisite.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy.egg-info/dependency_links.txt +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy.egg-info/entry_points.txt +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy.egg-info/requires.txt +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/src/cryptolabs_proxy.egg-info/top_level.txt +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_auth.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_edge_ownership.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_exporter_auth_protections.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_host_forwarding.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_identity_host_package.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_image_release_promotion.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_netbox_link.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_offline_image.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_proxy_lifecycle_review_repairs.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_release_identifier_boundary.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_release_publication.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_release_selection.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_release_workflow_temp_paths.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_routes.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_updates.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_version_consistency.py +0 -0
- {cryptolabs_proxy-1.3.1 → cryptolabs_proxy-1.4.0}/tests/test_vpm_prerequisite.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cryptolabs-proxy
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Unified reverse proxy and fleet management landing page for CryptoLabs products
|
|
5
5
|
Author-email: CryptoLabs <support@cryptolabs.co.za>
|
|
6
6
|
License: MIT
|
|
@@ -190,6 +190,10 @@ always-running internal VPM session and reauthentication endpoints cannot fall
|
|
|
190
190
|
through a preserved public <code>/auth/</code> route. Rollback restores the
|
|
191
191
|
original stored baseline exactly.
|
|
192
192
|
|
|
193
|
+
Supported DC monitoring adoption uses the private durable
|
|
194
|
+
[controller adoption guard](docs/controller-adoption-guard.md), including normal
|
|
195
|
+
updater and proxy-adoption exclusion during crash recovery.
|
|
196
|
+
|
|
193
197
|
### Vast Price Manager Fleet sign-in
|
|
194
198
|
|
|
195
199
|
### Vast Price Manager prerequisite
|
|
@@ -162,6 +162,10 @@ always-running internal VPM session and reauthentication endpoints cannot fall
|
|
|
162
162
|
through a preserved public <code>/auth/</code> route. Rollback restores the
|
|
163
163
|
original stored baseline exactly.
|
|
164
164
|
|
|
165
|
+
Supported DC monitoring adoption uses the private durable
|
|
166
|
+
[controller adoption guard](docs/controller-adoption-guard.md), including normal
|
|
167
|
+
updater and proxy-adoption exclusion during crash recovery.
|
|
168
|
+
|
|
165
169
|
### Vast Price Manager Fleet sign-in
|
|
166
170
|
|
|
167
171
|
### Vast Price Manager prerequisite
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cryptolabs-proxy"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.4.0"
|
|
8
8
|
description = "Unified reverse proxy and fleet management landing page for CryptoLabs products"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""Private controller exclusion on the updater's existing auth mount.
|
|
2
|
+
|
|
3
|
+
This module uses only stdlib so its fixed helper can run without auth startup,
|
|
4
|
+
network access, the Docker socket, or the source container's environment.
|
|
5
|
+
"""
|
|
6
|
+
import fcntl
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
import re
|
|
11
|
+
import socket
|
|
12
|
+
import stat
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
import uuid
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
|
|
18
|
+
MARKER = 'controller-adoption.json'
|
|
19
|
+
MAX_MARKER = 4096
|
|
20
|
+
GUARD_IDLE_SECONDS = 300
|
|
21
|
+
CAPABILITY = {'schema_version': 1, 'api': 'AdoptionEngine.controller_guard',
|
|
22
|
+
'durable_updater_exclusion': True}
|
|
23
|
+
FIELDS = {'schema_version', 'kind', 'operation_id', 'source_id', 'source_image',
|
|
24
|
+
'auth_mount_sha256', 'owner'}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class RecoveryError(RuntimeError):
|
|
28
|
+
"""Fixed public failure, without private filesystem or Engine details."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _hold():
|
|
32
|
+
raise RecoveryError('controller_recovery_required')
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def validate_record(value):
|
|
36
|
+
if not isinstance(value, dict) or set(value) != FIELDS:
|
|
37
|
+
_hold()
|
|
38
|
+
if type(value['schema_version']) is not int or value['schema_version'] != 1 or value['kind'] != 'dc-controller-adoption':
|
|
39
|
+
_hold()
|
|
40
|
+
patterns = {'operation_id': r'[A-Za-z0-9][A-Za-z0-9_.-]{0,95}',
|
|
41
|
+
'source_id': r'[0-9a-f]{64}',
|
|
42
|
+
'source_image': r'sha256:[0-9a-f]{64}',
|
|
43
|
+
'auth_mount_sha256': r'[0-9a-f]{64}', 'owner': r'[0-9a-f]{64}'}
|
|
44
|
+
for key, pattern in patterns.items():
|
|
45
|
+
if not isinstance(value[key], str) or not re.fullmatch(pattern, value[key]):
|
|
46
|
+
_hold()
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _private(info, *, directory=False):
|
|
51
|
+
return ((stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode))
|
|
52
|
+
and info.st_uid == os.geteuid()
|
|
53
|
+
and stat.S_IMODE(info.st_mode) == (0o700 if directory else 0o600)
|
|
54
|
+
and (directory or info.st_nlink == 1))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@contextmanager
|
|
58
|
+
def open_directory(directory):
|
|
59
|
+
"""Traverse each parent without following links; create only missing dirs."""
|
|
60
|
+
path = Path(directory)
|
|
61
|
+
if not path.is_absolute() or '..' in path.parts:
|
|
62
|
+
_hold()
|
|
63
|
+
descriptor = os.open('/', os.O_RDONLY | os.O_DIRECTORY)
|
|
64
|
+
try:
|
|
65
|
+
for index, part in enumerate(path.parts[1:]):
|
|
66
|
+
try:
|
|
67
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor)
|
|
68
|
+
except FileNotFoundError:
|
|
69
|
+
os.mkdir(part, 0o700, dir_fd=descriptor)
|
|
70
|
+
os.fsync(descriptor)
|
|
71
|
+
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=descriptor)
|
|
72
|
+
os.close(descriptor)
|
|
73
|
+
descriptor = child
|
|
74
|
+
if not _private(os.fstat(descriptor), directory=True):
|
|
75
|
+
_hold()
|
|
76
|
+
yield descriptor
|
|
77
|
+
except (OSError, ValueError):
|
|
78
|
+
_hold()
|
|
79
|
+
finally:
|
|
80
|
+
os.close(descriptor)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _unique_object(pairs):
|
|
84
|
+
result = {}
|
|
85
|
+
for key, value in pairs:
|
|
86
|
+
if key in result:
|
|
87
|
+
raise ValueError('duplicate field')
|
|
88
|
+
result[key] = value
|
|
89
|
+
return result
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def read_marker(directory_fd):
|
|
93
|
+
try:
|
|
94
|
+
descriptor = os.open(MARKER, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directory_fd)
|
|
95
|
+
except FileNotFoundError:
|
|
96
|
+
return None
|
|
97
|
+
except OSError:
|
|
98
|
+
_hold()
|
|
99
|
+
try:
|
|
100
|
+
info = os.fstat(descriptor)
|
|
101
|
+
if not _private(info) or info.st_size > MAX_MARKER:
|
|
102
|
+
_hold()
|
|
103
|
+
data = os.read(descriptor, MAX_MARKER + 1)
|
|
104
|
+
if len(data) > MAX_MARKER:
|
|
105
|
+
_hold()
|
|
106
|
+
try:
|
|
107
|
+
value = json.loads(data, object_pairs_hook=_unique_object)
|
|
108
|
+
except (ValueError, UnicodeError, RecursionError):
|
|
109
|
+
_hold()
|
|
110
|
+
return validate_record(value)
|
|
111
|
+
finally:
|
|
112
|
+
os.close(descriptor)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def require_clear(directory):
|
|
116
|
+
with open_directory(directory) as descriptor:
|
|
117
|
+
if read_marker(descriptor) is not None:
|
|
118
|
+
_hold()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _write_marker(directory_fd, value):
|
|
122
|
+
validate_record(value)
|
|
123
|
+
temporary = '.' + MARKER + '-' + uuid.uuid4().hex
|
|
124
|
+
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
125
|
+
0o600, dir_fd=directory_fd)
|
|
126
|
+
try:
|
|
127
|
+
with os.fdopen(descriptor, 'wb') as stream:
|
|
128
|
+
stream.write(json.dumps(value, sort_keys=True, separators=(',', ':')).encode())
|
|
129
|
+
stream.flush()
|
|
130
|
+
os.fsync(stream.fileno())
|
|
131
|
+
os.rename(temporary, MARKER, src_dir_fd=directory_fd, dst_dir_fd=directory_fd)
|
|
132
|
+
os.fsync(directory_fd)
|
|
133
|
+
finally:
|
|
134
|
+
try: os.unlink(temporary, dir_fd=directory_fd)
|
|
135
|
+
except FileNotFoundError: pass
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def open_lock(directory_fd, name):
|
|
139
|
+
descriptor = os.open(name, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
140
|
+
0o600, dir_fd=directory_fd)
|
|
141
|
+
if not _private(os.fstat(descriptor)):
|
|
142
|
+
os.close(descriptor)
|
|
143
|
+
_hold()
|
|
144
|
+
return descriptor
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@contextmanager
|
|
148
|
+
def updater_lock(directory, name):
|
|
149
|
+
with open_directory(directory) as directory_fd:
|
|
150
|
+
descriptor = None
|
|
151
|
+
try:
|
|
152
|
+
descriptor = open_lock(directory_fd, name)
|
|
153
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
|
154
|
+
yield
|
|
155
|
+
except OSError:
|
|
156
|
+
_hold()
|
|
157
|
+
finally:
|
|
158
|
+
if descriptor is not None:
|
|
159
|
+
os.close(descriptor)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def request(path, owner, action, timeout):
|
|
163
|
+
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client:
|
|
164
|
+
client.settimeout(timeout)
|
|
165
|
+
client.connect(path)
|
|
166
|
+
client.sendall(json.dumps({'owner': owner, 'action': action}).encode() + b'\n')
|
|
167
|
+
response = b''
|
|
168
|
+
while not response.endswith(b'\n'):
|
|
169
|
+
chunk = client.recv(1024)
|
|
170
|
+
if not chunk: _hold()
|
|
171
|
+
response += chunk
|
|
172
|
+
if len(response) > MAX_MARKER: _hold()
|
|
173
|
+
value = json.loads(response)
|
|
174
|
+
if value != {'owner': owner, 'state': 'completed' if action == 'complete' else 'held'}:
|
|
175
|
+
_hold()
|
|
176
|
+
return value
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# This fixed client is executed inside the exact guard helper, never the proxy.
|
|
180
|
+
CLIENT_COMMAND = """import json,socket,sys
|
|
181
|
+
with socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) as c:
|
|
182
|
+
c.settimeout(2); c.connect('/tmp/controller-guard.sock')
|
|
183
|
+
c.sendall(json.dumps({'owner':sys.argv[1],'action':sys.argv[2]}).encode()+b'\\n')
|
|
184
|
+
data=b''
|
|
185
|
+
while not data.endswith(b'\\n'):
|
|
186
|
+
part=c.recv(1024)
|
|
187
|
+
if not part: raise RuntimeError('guard_unavailable')
|
|
188
|
+
data+=part
|
|
189
|
+
if len(data)>4096: raise RuntimeError('guard_unavailable')
|
|
190
|
+
sys.stdout.buffer.write(data)
|
|
191
|
+
"""
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def helper_main(directory, socket_path, expected, recover, timeout):
|
|
195
|
+
"""Hold real locks until removal; durable marker outlives this process."""
|
|
196
|
+
validate_record(expected)
|
|
197
|
+
with open_directory(directory) as directory_fd:
|
|
198
|
+
descriptors = []
|
|
199
|
+
try:
|
|
200
|
+
deadline = time.monotonic() + timeout
|
|
201
|
+
for name in ('dispatch.lock', 'operation.lock'):
|
|
202
|
+
descriptor = open_lock(directory_fd, name)
|
|
203
|
+
descriptors.append((name, descriptor, os.fstat(descriptor)))
|
|
204
|
+
while True:
|
|
205
|
+
try:
|
|
206
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
207
|
+
break
|
|
208
|
+
except BlockingIOError:
|
|
209
|
+
if time.monotonic() >= deadline: _hold()
|
|
210
|
+
time.sleep(.02)
|
|
211
|
+
previous = read_marker(directory_fd)
|
|
212
|
+
if previous is not None:
|
|
213
|
+
if not recover or {k: v for k, v in previous.items() if k != 'owner'} != {k: v for k, v in expected.items() if k != 'owner'}:
|
|
214
|
+
_hold()
|
|
215
|
+
_write_marker(directory_fd, expected)
|
|
216
|
+
completed = False
|
|
217
|
+
|
|
218
|
+
def prove():
|
|
219
|
+
# Reopen by no-follow traversal to detect replaced parent/lock
|
|
220
|
+
# inodes. Fixed helper never unlocks its live descriptors.
|
|
221
|
+
with open_directory(directory) as current_fd:
|
|
222
|
+
if (os.fstat(current_fd).st_dev, os.fstat(current_fd).st_ino) != (os.fstat(directory_fd).st_dev, os.fstat(directory_fd).st_ino):
|
|
223
|
+
_hold()
|
|
224
|
+
for name, descriptor, original in descriptors:
|
|
225
|
+
current = os.stat(name, dir_fd=current_fd, follow_symlinks=False)
|
|
226
|
+
if not _private(current) or (current.st_dev, current.st_ino) != (original.st_dev, original.st_ino):
|
|
227
|
+
_hold()
|
|
228
|
+
if (os.fstat(descriptor).st_dev, os.fstat(descriptor).st_ino) != (original.st_dev, original.st_ino):
|
|
229
|
+
_hold()
|
|
230
|
+
actual = read_marker(current_fd)
|
|
231
|
+
if actual != (None if completed else expected): _hold()
|
|
232
|
+
|
|
233
|
+
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server:
|
|
234
|
+
server.bind(socket_path)
|
|
235
|
+
os.chmod(socket_path, 0o600)
|
|
236
|
+
server.listen(1)
|
|
237
|
+
verified_at = time.monotonic()
|
|
238
|
+
while True:
|
|
239
|
+
remaining = GUARD_IDLE_SECONDS - (time.monotonic() - verified_at)
|
|
240
|
+
if remaining <= 0:
|
|
241
|
+
return # Release locks; persisted marker stays in place.
|
|
242
|
+
server.settimeout(min(remaining, .25))
|
|
243
|
+
try:
|
|
244
|
+
connection, _ = server.accept()
|
|
245
|
+
except socket.timeout:
|
|
246
|
+
continue
|
|
247
|
+
with connection:
|
|
248
|
+
connection.settimeout(min(2, remaining))
|
|
249
|
+
try:
|
|
250
|
+
data = b''
|
|
251
|
+
while not data.endswith(b'\n'):
|
|
252
|
+
part = connection.recv(1024)
|
|
253
|
+
if not part: _hold()
|
|
254
|
+
data += part
|
|
255
|
+
if len(data) > MAX_MARKER: _hold()
|
|
256
|
+
command = json.loads(data)
|
|
257
|
+
if command not in ({'owner': expected['owner'], 'action': 'check'},
|
|
258
|
+
{'owner': expected['owner'], 'action': 'complete'}): _hold()
|
|
259
|
+
prove()
|
|
260
|
+
if time.monotonic() - verified_at >= GUARD_IDLE_SECONDS:
|
|
261
|
+
_hold()
|
|
262
|
+
verified_at = time.monotonic()
|
|
263
|
+
if command['action'] == 'complete':
|
|
264
|
+
if not completed:
|
|
265
|
+
os.unlink(MARKER, dir_fd=directory_fd)
|
|
266
|
+
os.fsync(directory_fd)
|
|
267
|
+
completed = True
|
|
268
|
+
state = 'completed'
|
|
269
|
+
else:
|
|
270
|
+
if completed: _hold()
|
|
271
|
+
state = 'held'
|
|
272
|
+
response = {'owner': expected['owner'], 'state': state}
|
|
273
|
+
except (OSError, ValueError, RecoveryError):
|
|
274
|
+
response = {'state': 'unavailable'}
|
|
275
|
+
try: connection.sendall(json.dumps(response).encode() + b'\n')
|
|
276
|
+
except OSError: pass
|
|
277
|
+
finally:
|
|
278
|
+
for _, descriptor, _ in descriptors: os.close(descriptor)
|
|
279
|
+
try: os.unlink(socket_path)
|
|
280
|
+
except FileNotFoundError: pass
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
if __name__ == '__main__':
|
|
284
|
+
helper_main(sys.argv[1], sys.argv[2], json.loads(sys.argv[3]), sys.argv[4] == 'recover', float(sys.argv[5]))
|
|
@@ -228,6 +228,8 @@ try:
|
|
|
228
228
|
fcntl.flock(stream,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
229
229
|
except BlockingIOError:
|
|
230
230
|
sys.exit(73)
|
|
231
|
+
# Presence is sufficient: malformed/unknown/link markers also require recovery.
|
|
232
|
+
if os.path.lexists(directory/'controller-adoption.json'): sys.exit(74)
|
|
231
233
|
pathlib.Path(sys.argv[2]).write_text('held')
|
|
232
234
|
while True: time.sleep(60)
|
|
233
235
|
"""
|
|
@@ -254,6 +256,151 @@ while True: time.sleep(60)
|
|
|
254
256
|
finally:
|
|
255
257
|
self.remove(name,ignore_missing=True)
|
|
256
258
|
|
|
259
|
+
@contextmanager
|
|
260
|
+
def controller_guard(self, source_proxy, image_id, operation_id, *, recover=False):
|
|
261
|
+
"""Exclude durable updates until verified controller completion.
|
|
262
|
+
|
|
263
|
+
Context exit always removes the helper but retains its marker. Recovery
|
|
264
|
+
reclaims the same source/mount/operation only under both real locks.
|
|
265
|
+
"""
|
|
266
|
+
from . import controller_recovery as recovery
|
|
267
|
+
from .migration import _mount_request
|
|
268
|
+
import uuid
|
|
269
|
+
if type(recover) is not bool:
|
|
270
|
+
raise AdoptionError('controller_guard_identity_invalid')
|
|
271
|
+
owner = uuid.uuid4().hex + uuid.uuid4().hex
|
|
272
|
+
mounts = source_proxy.get('Mounts', [])
|
|
273
|
+
auth = [m for m in mounts if m.get('Destination') == '/data/auth']
|
|
274
|
+
if len(auth) != 1 or not auth[0].get('RW', True) or any(
|
|
275
|
+
m.get('Destination', '').startswith('/data/auth/') for m in mounts):
|
|
276
|
+
raise AdoptionError('controller_guard_auth_mount_unsupported')
|
|
277
|
+
if auth[0].get('Type') not in ('bind', 'volume') or image_id != source_proxy.get('Image'):
|
|
278
|
+
raise AdoptionError('controller_guard_source_unsupported')
|
|
279
|
+
try:
|
|
280
|
+
mount = _mount_request(auth[0])
|
|
281
|
+
record = recovery.validate_record({'schema_version': 1, 'kind': 'dc-controller-adoption',
|
|
282
|
+
'operation_id': operation_id, 'source_id': source_proxy['Id'], 'source_image': image_id,
|
|
283
|
+
'auth_mount_sha256': sha256(_canonical(mount)), 'owner': owner})
|
|
284
|
+
except (KeyError, TypeError, ValueError, MigrationError, recovery.RecoveryError):
|
|
285
|
+
raise AdoptionError('controller_guard_identity_invalid') from None
|
|
286
|
+
name = 'cryptolabs-controller-lock-' + owner[:24]
|
|
287
|
+
command = Path(recovery.__file__).read_text()
|
|
288
|
+
self.create(name, {'Config': {'Image': image_id, 'Entrypoint': ['python3'],
|
|
289
|
+
'Cmd': ['-c', command, '/data/auth/update-jobs', '/tmp/controller-guard.sock',
|
|
290
|
+
json.dumps(record, sort_keys=True, separators=(',', ':')),
|
|
291
|
+
'recover' if recover else 'new', str(min(self.timeout, 15))],
|
|
292
|
+
'Healthcheck': {'Test': ['NONE']}, 'Labels': {'io.cryptolabs.controller.guard': owner,
|
|
293
|
+
'com.centurylinklabs.watchtower.enable': 'false'}},
|
|
294
|
+
'HostConfig': {'NetworkMode': 'none', 'RestartPolicy': {'Name': 'no'},
|
|
295
|
+
'Mounts': [mount], 'LogConfig': {'Type': 'json-file', 'Config': {'max-size': '1m', 'max-file': '1'}}},
|
|
296
|
+
'NetworkingConfig': {'EndpointsConfig': {}}})
|
|
297
|
+
helper_id = None
|
|
298
|
+
completed = False
|
|
299
|
+
|
|
300
|
+
def invoke(action):
|
|
301
|
+
try:
|
|
302
|
+
helper = self.inspect(helper_id or name)
|
|
303
|
+
if (not helper.get('State', {}).get('Running') or helper.get('Image') != image_id
|
|
304
|
+
or helper.get('Config', {}).get('Labels', {}).get('io.cryptolabs.controller.guard') != owner
|
|
305
|
+
or helper_id and helper.get('Id') != helper_id):
|
|
306
|
+
raise ValueError()
|
|
307
|
+
response = self._controller_guard_rpc(helper['Id'], owner, action)
|
|
308
|
+
if response != {'owner': owner, 'state': 'completed' if action == 'complete' else 'held'}:
|
|
309
|
+
raise ValueError()
|
|
310
|
+
return helper['Id']
|
|
311
|
+
except Exception:
|
|
312
|
+
raise AdoptionError('controller_guard_unavailable; controller recovery required') from None
|
|
313
|
+
|
|
314
|
+
def check():
|
|
315
|
+
if completed:
|
|
316
|
+
raise AdoptionError('controller_guard_completed')
|
|
317
|
+
invoke('check')
|
|
318
|
+
|
|
319
|
+
def complete():
|
|
320
|
+
nonlocal completed
|
|
321
|
+
invoke('complete')
|
|
322
|
+
completed = True
|
|
323
|
+
|
|
324
|
+
check.complete = complete
|
|
325
|
+
try:
|
|
326
|
+
self.start(name)
|
|
327
|
+
deadline = time.monotonic() + min(self.timeout, 15) + 1
|
|
328
|
+
while True:
|
|
329
|
+
try:
|
|
330
|
+
helper_id = invoke('check')
|
|
331
|
+
break
|
|
332
|
+
except AdoptionError:
|
|
333
|
+
if time.monotonic() >= deadline or not self.inspect(name).get('State', {}).get('Running'):
|
|
334
|
+
raise AdoptionError('controller_guard_unavailable; controller recovery required') from None
|
|
335
|
+
time.sleep(.02)
|
|
336
|
+
yield check
|
|
337
|
+
finally:
|
|
338
|
+
self.remove(helper_id or name, ignore_missing=True)
|
|
339
|
+
|
|
340
|
+
def _controller_guard_rpc(self, helper_id, owner, action):
|
|
341
|
+
"""Execute the fixed client on this Engine, with bounded framed output."""
|
|
342
|
+
from .controller_recovery import CLIENT_COMMAND
|
|
343
|
+
from .migration import _UnixHTTPConnection
|
|
344
|
+
try:
|
|
345
|
+
command = ['python3', '-c', CLIENT_COMMAND, owner, action]
|
|
346
|
+
if action not in ('check', 'complete'):
|
|
347
|
+
raise ValueError()
|
|
348
|
+
created = self._request('POST', '/containers/' + quote(helper_id, safe='') + '/exec',
|
|
349
|
+
{'AttachStdin': False, 'AttachStdout': True, 'AttachStderr': True,
|
|
350
|
+
'Tty': False, 'Cmd': command})
|
|
351
|
+
if len(created) > 4096:
|
|
352
|
+
raise ValueError()
|
|
353
|
+
exec_id = json.loads(created)['Id']
|
|
354
|
+
if not isinstance(exec_id, str) or not re.fullmatch('[0-9a-f]{64}', exec_id):
|
|
355
|
+
raise ValueError()
|
|
356
|
+
# Generic Engine requests read the complete response. Bound this
|
|
357
|
+
# attached exec stream before allocation on the very same socket.
|
|
358
|
+
connection = _UnixHTTPConnection(self.socket_path, min(self.timeout, 5))
|
|
359
|
+
try:
|
|
360
|
+
connection.request('POST', '/exec/' + exec_id + '/start',
|
|
361
|
+
body=json.dumps({'Detach': False, 'Tty': False}).encode(),
|
|
362
|
+
headers={'Content-Type': 'application/json'})
|
|
363
|
+
response = connection.getresponse()
|
|
364
|
+
if response.status != 200:
|
|
365
|
+
raise ValueError()
|
|
366
|
+
output = response.read(16385)
|
|
367
|
+
finally:
|
|
368
|
+
connection.close()
|
|
369
|
+
if len(output) > 16384:
|
|
370
|
+
raise ValueError()
|
|
371
|
+
stdout = bytearray()
|
|
372
|
+
offset = 0
|
|
373
|
+
while offset < len(output):
|
|
374
|
+
header = output[offset:offset + 8]
|
|
375
|
+
if len(header) != 8 or header[0] not in (1, 2) or header[1:4] != b'\0\0\0':
|
|
376
|
+
raise ValueError()
|
|
377
|
+
length = int.from_bytes(header[4:8], 'big')
|
|
378
|
+
offset += 8
|
|
379
|
+
if length > len(output) - offset or header[0] == 2 and length:
|
|
380
|
+
raise ValueError()
|
|
381
|
+
if header[0] == 1:
|
|
382
|
+
if len(stdout) + length > 4096:
|
|
383
|
+
raise ValueError()
|
|
384
|
+
stdout.extend(output[offset:offset + length])
|
|
385
|
+
offset += length
|
|
386
|
+
inspected = self._request('GET', '/exec/' + exec_id + '/json')
|
|
387
|
+
if len(inspected) > 4096:
|
|
388
|
+
raise ValueError()
|
|
389
|
+
execution = json.loads(inspected)
|
|
390
|
+
process = execution.get('ProcessConfig', {})
|
|
391
|
+
if (execution.get('ID') != exec_id or execution.get('ContainerID') != helper_id
|
|
392
|
+
or execution.get('Running') is not False
|
|
393
|
+
or type(execution.get('ExitCode')) is not int or execution['ExitCode'] != 0
|
|
394
|
+
or process.get('entrypoint') != command[0] or process.get('arguments') != command[1:]
|
|
395
|
+
or process.get('tty') is not False):
|
|
396
|
+
raise ValueError()
|
|
397
|
+
value = json.loads(stdout)
|
|
398
|
+
if value != {'owner': owner, 'state': 'completed' if action == 'complete' else 'held'}:
|
|
399
|
+
raise ValueError()
|
|
400
|
+
return value
|
|
401
|
+
except Exception:
|
|
402
|
+
raise AdoptionError('controller_guard_unavailable') from None
|
|
403
|
+
|
|
257
404
|
def read_config(self,name):
|
|
258
405
|
return CustomConfigMigrator(Path('/etc/cryptolabs-proxy'),Path('/etc/cryptolabs-proxy/migrations'),engine=self)._read_active_config(name)
|
|
259
406
|
def image(self,reference): return json.loads(self._request('GET','/images/'+quote(reference,safe='')+'/json'))
|
|
@@ -22,6 +22,7 @@ from urllib.request import urlopen
|
|
|
22
22
|
|
|
23
23
|
from .migration import DockerEngine, MigrationError, _mount_request, endpoint_configurations
|
|
24
24
|
from . import release_selection as selection
|
|
25
|
+
from . import controller_recovery
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
SERVICES = {
|
|
@@ -55,7 +56,8 @@ PROXY_ADOPTION_MESSAGE = ('Proxy replacement requires the published host CLI: ru
|
|
|
55
56
|
def get_capabilities():
|
|
56
57
|
"""Additive caller contract; proxy replacement requires host authority."""
|
|
57
58
|
return {'selection_contract': selection.SELECTION_CONTRACT,
|
|
58
|
-
'proxy_replacement_policy': PROXY_REPLACEMENT_POLICY
|
|
59
|
+
'proxy_replacement_policy': PROXY_REPLACEMENT_POLICY,
|
|
60
|
+
'controller_adoption_guard': dict(controller_recovery.CAPABILITY)}
|
|
59
61
|
|
|
60
62
|
|
|
61
63
|
def _require_selected_updater(engine, source):
|
|
@@ -436,11 +438,11 @@ def job_status(job_id, *, directory=JOBS_DIR, engine=None):
|
|
|
436
438
|
|
|
437
439
|
@contextmanager
|
|
438
440
|
def _lock(directory, name):
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
441
|
+
try:
|
|
442
|
+
with controller_recovery.updater_lock(directory, name):
|
|
443
|
+
yield
|
|
444
|
+
except controller_recovery.RecoveryError:
|
|
445
|
+
raise UpdateError('controller_recovery_required') from None
|
|
444
446
|
|
|
445
447
|
|
|
446
448
|
def helper_request(source, directory, job_id):
|
|
@@ -476,6 +478,10 @@ def submit_job(service, branch, action, *, directory=JOBS_DIR, engine=None,
|
|
|
476
478
|
if snapshot and service != 'all' and (service not in targets or targets[service]['owner'] != 'proxy-updater'):
|
|
477
479
|
raise UpdateError('service_not_selected')
|
|
478
480
|
with _lock(directory, 'dispatch.lock'):
|
|
481
|
+
try:
|
|
482
|
+
controller_recovery.require_clear(directory)
|
|
483
|
+
except controller_recovery.RecoveryError:
|
|
484
|
+
raise UpdateError('controller_recovery_required') from None
|
|
479
485
|
active = Path(directory) / 'active.json'
|
|
480
486
|
if active.exists():
|
|
481
487
|
previous = read_job(directory, json.loads(active.read_text())['id'])
|
|
@@ -614,6 +620,13 @@ def run_job(directory, job_id, *, engine=None):
|
|
|
614
620
|
job = read_job(directory, job_id)
|
|
615
621
|
if job.get('state') not in (None, 'queued'):
|
|
616
622
|
return job # A completed/interrupted operation is never replayed.
|
|
623
|
+
try:
|
|
624
|
+
controller_recovery.require_clear(directory)
|
|
625
|
+
except controller_recovery.RecoveryError:
|
|
626
|
+
job.update(state='blocked', success=False, reason='controller_recovery_required',
|
|
627
|
+
finished_at=time.time(), results={})
|
|
628
|
+
write_job(directory, job)
|
|
629
|
+
return job
|
|
617
630
|
job.update(state='running', results={})
|
|
618
631
|
write_job(directory, job)
|
|
619
632
|
if 'selection_snapshot' in job:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cryptolabs-proxy
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.4.0
|
|
4
4
|
Summary: Unified reverse proxy and fleet management landing page for CryptoLabs products
|
|
5
5
|
Author-email: CryptoLabs <support@cryptolabs.co.za>
|
|
6
6
|
License: MIT
|
|
@@ -190,6 +190,10 @@ always-running internal VPM session and reauthentication endpoints cannot fall
|
|
|
190
190
|
through a preserved public <code>/auth/</code> route. Rollback restores the
|
|
191
191
|
original stored baseline exactly.
|
|
192
192
|
|
|
193
|
+
Supported DC monitoring adoption uses the private durable
|
|
194
|
+
[controller adoption guard](docs/controller-adoption-guard.md), including normal
|
|
195
|
+
updater and proxy-adoption exclusion during crash recovery.
|
|
196
|
+
|
|
193
197
|
### Vast Price Manager Fleet sign-in
|
|
194
198
|
|
|
195
199
|
### Vast Price Manager prerequisite
|
|
@@ -5,6 +5,7 @@ src/cryptolabs_proxy/__init__.py
|
|
|
5
5
|
src/cryptolabs_proxy/auth.py
|
|
6
6
|
src/cryptolabs_proxy/cli.py
|
|
7
7
|
src/cryptolabs_proxy/config.py
|
|
8
|
+
src/cryptolabs_proxy/controller_recovery.py
|
|
8
9
|
src/cryptolabs_proxy/custom_config.py
|
|
9
10
|
src/cryptolabs_proxy/exporter_manager.py
|
|
10
11
|
src/cryptolabs_proxy/identity.py
|
|
@@ -34,6 +35,8 @@ src/cryptolabs_proxy/templates/standard-service-routes.conf.j2
|
|
|
34
35
|
src/cryptolabs_proxy/templates/vast-price-manager-auth-blocks.conf.j2
|
|
35
36
|
src/cryptolabs_proxy/templates/vast-price-manager.conf.j2
|
|
36
37
|
tests/test_auth.py
|
|
38
|
+
tests/test_controller_guard_transport.py
|
|
39
|
+
tests/test_controller_recovery_guard.py
|
|
37
40
|
tests/test_edge_ownership.py
|
|
38
41
|
tests/test_exporter_auth_protections.py
|
|
39
42
|
tests/test_host_forwarding.py
|