msgctl 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- msg/__init__.py +2 -0
- msg/admin/__init__.py +0 -0
- msg/admin/backup_retirement.py +56 -0
- msg/admin/backups.py +341 -0
- msg/admin/custodial_check.py +77 -0
- msg/admin/diagnostics.py +750 -0
- msg/admin/market.py +91 -0
- msg/admin/market_check.py +341 -0
- msg/admin/money.py +322 -0
- msg/admin/preflight.py +142 -0
- msg/admin/recovery_replay.py +281 -0
- msg/admin/restore_database.py +37 -0
- msg/admin/root.py +347 -0
- msg/admin/rotation.py +197 -0
- msg/admin/token_delivery_check.py +54 -0
- msg/admin/upgrade_check.py +58 -0
- msg/application.py +163 -0
- msg/bootstrap.py +305 -0
- msg/cli.py +537 -0
- msg/client.py +725 -0
- msg/client_certificates.py +98 -0
- msg/client_content.py +43 -0
- msg/client_custodial.py +206 -0
- msg/client_market.py +104 -0
- msg/client_recovery.py +186 -0
- msg/client_secrets.py +58 -0
- msg/client_tokens.py +99 -0
- msg/client_upgrade.py +188 -0
- msg/config.py +318 -0
- msg/constants.py +11 -0
- msg/core/__init__.py +0 -0
- msg/core/batching.py +23 -0
- msg/core/codec.py +253 -0
- msg/core/contracts.py +128 -0
- msg/core/cursors.py +68 -0
- msg/core/email_address.py +33 -0
- msg/core/errors.py +22 -0
- msg/core/events.py +6 -0
- msg/core/execution_ports.py +20 -0
- msg/core/executor.py +193 -0
- msg/core/models.py +517 -0
- msg/core/packet.py +71 -0
- msg/core/permissions.py +14 -0
- msg/core/query.py +44 -0
- msg/core/read_query.py +60 -0
- msg/core/registry.py +173 -0
- msg/core/requests.py +58 -0
- msg/core/schema_policy.py +28 -0
- msg/core/schemas.py +30 -0
- msg/core/tags.py +21 -0
- msg/core/template_dsl.py +94 -0
- msg/core/text_patch.py +283 -0
- msg/core/tool_execution.py +28 -0
- msg/core/transfer.py +183 -0
- msg/daemon.py +255 -0
- msg/data/__init__.py +0 -0
- msg/data/bootstrap.json +72 -0
- msg/data/favicon.png +0 -0
- msg/data/logo-dark.svg +5 -0
- msg/data/logo.svg +5 -0
- msg/data/recovery-checkpoint.example.json +1 -0
- msg/data/recovery-checkpoint.schema.json +118 -0
- msg/data/shortcodes.json +1 -0
- msg/data/system/AGENTS.md +6 -0
- msg/data/system/rules/_index.md +17 -0
- msg/data/system/rules/auth.md +6 -0
- msg/data/system/rules/files.md +6 -0
- msg/data/system/rules/identity.md +6 -0
- msg/data/system/rules/protocol.md +6 -0
- msg/data/system/rules/read-write.md +6 -0
- msg/data/system/rules/recovery.md +6 -0
- msg/data/system/rules/security.md +10 -0
- msg/data/system/rules/topics.md +6 -0
- msg/extensions/__init__.py +1 -0
- msg/extensions/hosting.py +259 -0
- msg/extensions/keystore.py +96 -0
- msg/extensions/repositories.py +769 -0
- msg/extensions/rss.py +67 -0
- msg/extensions/ssh.py +264 -0
- msg/extensions/ssh_git.py +300 -0
- msg/extensions/tools.py +150 -0
- msg/hosting_runtime.py +138 -0
- msg/market/__init__.py +1 -0
- msg/market/arbitration.py +481 -0
- msg/market/delivery.py +338 -0
- msg/market/delivery_notifications.py +110 -0
- msg/market/delivery_targets.py +76 -0
- msg/market/email.py +100 -0
- msg/market/escrow.py +345 -0
- msg/market/orders.py +218 -0
- msg/market/policy.py +147 -0
- msg/market/rationale.py +91 -0
- msg/market/references.py +19 -0
- msg/market/targets.py +157 -0
- msg/plugins/__init__.py +28 -0
- msg/plugins/achievements.py +316 -0
- msg/plugins/batch.py +32 -0
- msg/plugins/bounty.py +366 -0
- msg/plugins/collaboration.py +261 -0
- msg/plugins/common.py +217 -0
- msg/plugins/communication.py +853 -0
- msg/plugins/content.py +855 -0
- msg/plugins/custodial_lifecycle.py +224 -0
- msg/plugins/delivery.py +230 -0
- msg/plugins/discovery.py +1266 -0
- msg/plugins/discussion.py +162 -0
- msg/plugins/extensions.py +10 -0
- msg/plugins/following.py +78 -0
- msg/plugins/hosting_capacity.py +52 -0
- msg/plugins/identity.py +1654 -0
- msg/plugins/money.py +221 -0
- msg/plugins/offers.py +183 -0
- msg/plugins/orders.py +273 -0
- msg/plugins/recovery.py +376 -0
- msg/plugins/schemas.py +5 -0
- msg/plugins/sharing.py +300 -0
- msg/plugins/store.py +249 -0
- msg/plugins/system.py +75 -0
- msg/plugins/transfer.py +249 -0
- msg/py.typed +0 -0
- msg/security/__init__.py +0 -0
- msg/security/age_keys.py +112 -0
- msg/security/authentication.py +141 -0
- msg/security/authorization.py +300 -0
- msg/security/backup_retirement.py +135 -0
- msg/security/capabilities.py +157 -0
- msg/security/certificates.py +183 -0
- msg/security/crypto.py +102 -0
- msg/security/custodial_migration.py +292 -0
- msg/security/custody_history.py +146 -0
- msg/security/network.py +81 -0
- msg/security/policy.py +67 -0
- msg/security/quarantine.py +19 -0
- msg/security/root_files.py +67 -0
- msg/security/rotation_journal.py +60 -0
- msg/security/sealed_box.py +70 -0
- msg/security/sharing_policy.py +29 -0
- msg/security/token_delivery.py +93 -0
- msg/security/vault.py +186 -0
- msg/storage/__init__.py +0 -0
- msg/storage/capacity.py +100 -0
- msg/storage/custodial_migration.py +22 -0
- msg/storage/git.py +435 -0
- msg/storage/ledger_migration.py +307 -0
- msg/storage/market_migration.py +49 -0
- msg/storage/postgres.py +663 -0
- msg/storage/query.py +39 -0
- msg/storage/read_only.py +126 -0
- msg/storage/session.py +403 -0
- msg/storage/sqlite.py +360 -0
- msg/storage/topic_event_migration.py +26 -0
- msg/storage/valkey_bus.py +45 -0
- msg/transports/__init__.py +0 -0
- msg/transports/client.py +183 -0
- msg/transports/dictionary.py +387 -0
- msg/transports/graphql.py +75 -0
- msg/transports/http.py +72 -0
- msg/transports/http_routes.py +1577 -0
- msg/transports/mcp.py +127 -0
- msg/transports/packet.py +42 -0
- msg/transports/read_tree_path.py +65 -0
- msg/transports/stdio.py +36 -0
- msg/transports/url_safety.py +123 -0
- msg/tui.py +368 -0
- msg/workers/__init__.py +1 -0
- msg/workers/effects.py +406 -0
- msg/workers/leases.py +19 -0
- msg/workers/mail.py +63 -0
- msg/workers/maintenance.py +310 -0
- msg/workers/sandbox.py +74 -0
- msg/workers/sandbox_child.py +189 -0
- msg/workers/webhook.py +160 -0
- msgctl-0.1.0a1.dist-info/METADATA +96 -0
- msgctl-0.1.0a1.dist-info/RECORD +176 -0
- msgctl-0.1.0a1.dist-info/WHEEL +4 -0
- msgctl-0.1.0a1.dist-info/entry_points.txt +4 -0
msg/client.py
ADDED
|
@@ -0,0 +1,725 @@
|
|
|
1
|
+
"""Local identity, automatic signing, resumable file exchange, and bounded retries."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
from datetime import UTC, datetime, timedelta
|
|
6
|
+
import hashlib
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
import re
|
|
10
|
+
import stat
|
|
11
|
+
from urllib.parse import urlsplit
|
|
12
|
+
from uuid import uuid4
|
|
13
|
+
|
|
14
|
+
import httpx
|
|
15
|
+
|
|
16
|
+
from msg.core.codec import b64, unb64, canonical, digest, loads, wire, decode, parse_time
|
|
17
|
+
from msg.core.errors import Failure, require
|
|
18
|
+
from msg.core.models import ResourceRef, Certificate, Signature
|
|
19
|
+
from msg.core.requests import request_for
|
|
20
|
+
from msg.security.crypto import Ed25519Signer, subject_id, key_id
|
|
21
|
+
from msg.security.age_keys import (generate_age_key,recipient_from_identity,
|
|
22
|
+
public_from_recipient,encryption_key_id)
|
|
23
|
+
from msg.security.vault import client_upgrade_proof
|
|
24
|
+
from msg.storage.git import durable_write
|
|
25
|
+
from msg.client_tokens import read_journal, remove_journal, token_operation
|
|
26
|
+
from msg.transports.client import HTTPTransport, GraphQLTransport, MCPHTTPTransport
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def hash_file(path):
|
|
30
|
+
hashed = hashlib.sha256()
|
|
31
|
+
size = 0
|
|
32
|
+
with Path(path).open('rb') as stream:
|
|
33
|
+
while data := stream.read(65536):
|
|
34
|
+
size += len(data)
|
|
35
|
+
hashed.update(data)
|
|
36
|
+
return size, 'sha256:'+hashed.hexdigest()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ClientState:
|
|
40
|
+
"""Owned files only; a failed registration never loses its private key."""
|
|
41
|
+
def __init__(self, directory=None, *, server=None):
|
|
42
|
+
self.directory = Path(directory or Path.home()/'.config'/'msg')
|
|
43
|
+
self.directory.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
44
|
+
require(not self.directory.is_symlink(),'unsafe_client_directory')
|
|
45
|
+
require(self.directory.stat().st_uid==os.geteuid(),'client_directory_not_owned')
|
|
46
|
+
self.directory.chmod(0o700)
|
|
47
|
+
self.path = self.directory/'client.json'
|
|
48
|
+
self.key_path = self.directory/'identity.key'
|
|
49
|
+
self.age_key_path = self.directory/'encryption.agekey'
|
|
50
|
+
self.pending_path = self.directory/'registration.json'
|
|
51
|
+
self.data = loads(self.path.read_bytes()) if self.path.exists() else {'version':1}
|
|
52
|
+
require(self.data.get('version')==1,'unknown_client_state_version')
|
|
53
|
+
require(server is None or self.data.get('server',server)==server.rstrip('/'),'client_server_mismatch')
|
|
54
|
+
self.server = self.data.get('server',server or 'https://msg.lmm.best').rstrip('/')
|
|
55
|
+
self.data['server'] = self.server
|
|
56
|
+
self.signer = None
|
|
57
|
+
self.encryption_recipient = None
|
|
58
|
+
if self.key_path.exists():
|
|
59
|
+
require(self.key_path.is_file() and not self.key_path.is_symlink() and
|
|
60
|
+
self.key_path.stat().st_uid==os.geteuid() and self.key_path.stat().st_mode&0o077==0,
|
|
61
|
+
'unsafe_client_key_permissions')
|
|
62
|
+
self.signer = Ed25519Signer.from_bytes(self.key_path.read_bytes())
|
|
63
|
+
if self.age_key_path.exists():
|
|
64
|
+
require(self.age_key_path.is_file() and not self.age_key_path.is_symlink() and
|
|
65
|
+
self.age_key_path.stat().st_uid==os.geteuid() and
|
|
66
|
+
self.age_key_path.stat().st_mode&0o077==0,'unsafe_client_key_permissions')
|
|
67
|
+
self.encryption_recipient=recipient_from_identity(self.age_key_path.read_text().strip())
|
|
68
|
+
self._save()
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def subject(self):
|
|
72
|
+
return self.data.get('subject_id')
|
|
73
|
+
|
|
74
|
+
@property
|
|
75
|
+
def certificates(self):
|
|
76
|
+
return tuple(self.data.get('certificates',()))
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def token(self):
|
|
80
|
+
saved = self.data.get('token')
|
|
81
|
+
return (saved['credential_id'],unb64(saved['value'])) if saved else None
|
|
82
|
+
|
|
83
|
+
def _save(self):
|
|
84
|
+
durable_write(self.path,canonical(self.data),mode=0o600)
|
|
85
|
+
|
|
86
|
+
def save_signer(self, signer):
|
|
87
|
+
require(not self.key_path.exists(),'identity_key_already_exists')
|
|
88
|
+
durable_write(self.key_path,signer.private_bytes(),mode=0o600)
|
|
89
|
+
self.signer = signer
|
|
90
|
+
|
|
91
|
+
def ensure_encryption_key(self):
|
|
92
|
+
if self.encryption_recipient is None:
|
|
93
|
+
require(not self.age_key_path.exists(),'invalid_age_identity')
|
|
94
|
+
identity,recipient=generate_age_key()
|
|
95
|
+
durable_write(self.age_key_path,(identity+'\n').encode(),mode=0o600)
|
|
96
|
+
self.encryption_recipient=recipient
|
|
97
|
+
return self.encryption_recipient
|
|
98
|
+
|
|
99
|
+
def accept_identity(self, result):
|
|
100
|
+
require(result.status=='ok','identity_operation_failed')
|
|
101
|
+
if result.data.get('encryption_recipient'):
|
|
102
|
+
if result.operation=='identity.custodial_create':
|
|
103
|
+
require(self.signer is None and self.encryption_recipient is None,
|
|
104
|
+
'custodial_client_key_conflict')
|
|
105
|
+
self.data['custodial_encryption_recipient']=result.data['encryption_recipient']
|
|
106
|
+
else:
|
|
107
|
+
require(result.data['encryption_recipient']==self.encryption_recipient,
|
|
108
|
+
'encryption_recipient_mismatch')
|
|
109
|
+
self.data.pop('custodial_encryption_recipient',None)
|
|
110
|
+
self.data['subject_id'] = result.subject
|
|
111
|
+
if result.data.get('certificate_id'):
|
|
112
|
+
self.data['certificates'] = [result.data['certificate_id']]
|
|
113
|
+
if result.data.get('token'):
|
|
114
|
+
self.data['token'] = {'credential_id':result.data['credential_id'], 'value':result.data['token'],
|
|
115
|
+
'expires_at':result.data['expires_at']}
|
|
116
|
+
elif self.signer is not None:
|
|
117
|
+
self.data.pop('token',None)
|
|
118
|
+
self._save()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class MsgClient:
|
|
122
|
+
def __init__(self, state, transport, *, clock=None, retries=2):
|
|
123
|
+
self.state, self.transport = state, transport
|
|
124
|
+
self.clock = clock or (lambda:datetime.now(UTC))
|
|
125
|
+
self.retries = retries
|
|
126
|
+
require(state.server==transport.server,'client_server_mismatch')
|
|
127
|
+
|
|
128
|
+
def prepare(self, operation, arguments, *, expected=(), request_id=None, return_fields=(), subject=None,
|
|
129
|
+
signer=None, certificates=None, anonymous=False, contract_version=1,expires_at=None):
|
|
130
|
+
require(not operation.startswith('root.'),'local_only')
|
|
131
|
+
if (self.state.directory/'token-rotation.json').exists():
|
|
132
|
+
require(operation=='identity.token_rotate','token_rotation_pending')
|
|
133
|
+
selected_signer = signer or self.state.signer
|
|
134
|
+
selected_subject = subject or self.state.subject
|
|
135
|
+
# An explicit temporary token remains authoritative until upgrade succeeds.
|
|
136
|
+
token = self.state.token if signer is None else None
|
|
137
|
+
if token:
|
|
138
|
+
selected_signer = None
|
|
139
|
+
return request_for(operation,arguments,self.state.server,
|
|
140
|
+
subject=None if anonymous else selected_subject,signer=None if anonymous else selected_signer,
|
|
141
|
+
token=None if anonymous else token, certificates=() if anonymous else
|
|
142
|
+
(self.state.certificates if certificates is None else certificates),request_id=request_id,
|
|
143
|
+
expires_at=expires_at or self.clock()+timedelta(seconds=180),source='msg',expected=expected,
|
|
144
|
+
return_fields=return_fields,contract_version=contract_version)
|
|
145
|
+
|
|
146
|
+
async def send(self, packet):
|
|
147
|
+
result = None
|
|
148
|
+
for attempt in range(self.retries+1):
|
|
149
|
+
try:
|
|
150
|
+
result = await self.transport.call(packet)
|
|
151
|
+
if result.status!='error' or not result.error.retryable or attempt==self.retries:
|
|
152
|
+
return result
|
|
153
|
+
except (httpx.TimeoutException,httpx.NetworkError) as exc:
|
|
154
|
+
if attempt==self.retries:
|
|
155
|
+
# Transport failure is not evidence of a rollback. The caller
|
|
156
|
+
# must retry this request ID, not invent another operation.
|
|
157
|
+
raise Failure('transport_uncertain',retryable=True,
|
|
158
|
+
details={'request_id':packet.request_id}) from exc
|
|
159
|
+
except Failure as exc:
|
|
160
|
+
if not exc.retryable or attempt==self.retries:
|
|
161
|
+
raise
|
|
162
|
+
await asyncio.sleep(min(2,0.2*(2**attempt)))
|
|
163
|
+
return result
|
|
164
|
+
|
|
165
|
+
async def call(self, operation, arguments=None, **kwargs):
|
|
166
|
+
return await self.send(self.prepare(operation,arguments or {},**kwargs))
|
|
167
|
+
|
|
168
|
+
async def _website_generation(self, website):
|
|
169
|
+
meta=self.checked(await self.call('discovery.get',{'id':website,'view':'meta'}))
|
|
170
|
+
require(meta.data.get('type')=='website','not_a_website')
|
|
171
|
+
return meta.data['id'],meta.data['generation']
|
|
172
|
+
|
|
173
|
+
@staticmethod
|
|
174
|
+
def _hosting_entries(entries):
|
|
175
|
+
require(type(entries) is list and bool(entries),'hosting_entries_required')
|
|
176
|
+
names=set();validated=[]
|
|
177
|
+
for item in entries:
|
|
178
|
+
require(type(item) is dict and set(item)=={'path','source'},'invalid_hosting_entry')
|
|
179
|
+
path=item['path'];source=item['source']
|
|
180
|
+
require(type(path) is str and 0<len(path)<=2048 and
|
|
181
|
+
all(p not in {'','.','..'} and not p.startswith('.') for p in path.split('/')) and
|
|
182
|
+
'\\' not in path and all(32<=ord(c)<127 for c in path) and path not in names,
|
|
183
|
+
'invalid_hosting_path')
|
|
184
|
+
require(type(source) is dict and type(source.get('id')) is str and
|
|
185
|
+
type(source.get('revision')) is str and bool(source['revision']) and
|
|
186
|
+
set(source)=={'id','revision'},'source_revision_required')
|
|
187
|
+
names.add(path);validated.append({'path':path,'source':source})
|
|
188
|
+
return validated
|
|
189
|
+
|
|
190
|
+
async def hosting_preview(self,website,entries):
|
|
191
|
+
require(self.state.signer is not None and self.state.token is None,
|
|
192
|
+
'signing_identity_required')
|
|
193
|
+
rid,generation=await self._website_generation(website)
|
|
194
|
+
return await self.call('hosting.preview',{'id':rid,'entries':self._hosting_entries(entries)},
|
|
195
|
+
expected=((rid,generation),))
|
|
196
|
+
|
|
197
|
+
async def hosting_deploy(self,website,entries):
|
|
198
|
+
require(self.state.signer is not None and self.state.token is None,
|
|
199
|
+
'signing_identity_required')
|
|
200
|
+
rid,generation=await self._website_generation(website)
|
|
201
|
+
return await self.call('hosting.deploy',{'id':rid,'entries':self._hosting_entries(entries)},
|
|
202
|
+
expected=((rid,generation),))
|
|
203
|
+
|
|
204
|
+
async def hosting_activate(self,website,revision):
|
|
205
|
+
require(self.state.signer is not None and self.state.token is None,
|
|
206
|
+
'signing_identity_required')
|
|
207
|
+
require(type(revision) is str and bool(revision),'revision_required')
|
|
208
|
+
rid,generation=await self._website_generation(website)
|
|
209
|
+
return await self.call('hosting.activate',{'id':rid,'revision':revision},
|
|
210
|
+
expected=((rid,generation),))
|
|
211
|
+
|
|
212
|
+
async def hosting_preview_get(self,site_path,candidate_id,file_path='index.html',*,max_bytes=1048576):
|
|
213
|
+
"""Fetch candidate bytes with a header-bound proof, never a shareable URL."""
|
|
214
|
+
require(self.state.signer is not None and self.state.token is None,
|
|
215
|
+
'signing_identity_required')
|
|
216
|
+
require(type(self.transport) is HTTPTransport,'hosting_http_transport_required')
|
|
217
|
+
require(type(site_path) is str and re.fullmatch(r'/[@&][A-Za-z0-9_-]+/[A-Za-z0-9_-]+',site_path),
|
|
218
|
+
'invalid_website_path')
|
|
219
|
+
require(type(candidate_id) is str and re.fullmatch(r'[A-Za-z0-9_-]{1,128}',candidate_id),
|
|
220
|
+
'invalid_candidate_id')
|
|
221
|
+
require(type(file_path) is str and file_path and len(file_path)<=2048 and
|
|
222
|
+
all(p not in {'','.','..'} and not p.startswith('.') for p in file_path.split('/')) and
|
|
223
|
+
all(32<=ord(c)<127 for c in file_path) and '\\' not in file_path and
|
|
224
|
+
'%' not in file_path and '?' not in file_path and '#' not in file_path,
|
|
225
|
+
'invalid_hosting_path')
|
|
226
|
+
require(type(max_bytes) is int and 0<max_bytes<=10485760,'invalid_preview_limit')
|
|
227
|
+
packet=self.prepare('discovery.raw',{'id':candidate_id})
|
|
228
|
+
header=b64(canonical(wire(packet)))
|
|
229
|
+
path=site_path+'/_preview/'+candidate_id+'/'+file_path
|
|
230
|
+
async with self.transport.http.stream('GET',self.state.server+path,
|
|
231
|
+
headers={'X-Msg-Request':header},follow_redirects=False) as response:
|
|
232
|
+
require(not response.is_redirect,'redirect_not_allowed')
|
|
233
|
+
if response.status_code!=200:
|
|
234
|
+
code='preview_permission_denied' if response.status_code in {401,403} else \
|
|
235
|
+
'preview_not_found' if response.status_code==404 else 'preview_request_failed'
|
|
236
|
+
raise Failure(code,details={'status':response.status_code})
|
|
237
|
+
csp=response.headers.get('content-security-policy','').lower()
|
|
238
|
+
require('sandbox' in csp.split(';')[0].split() and
|
|
239
|
+
'allow-scripts' not in csp and 'allow-same-origin' not in csp and
|
|
240
|
+
response.headers.get('x-content-type-options')=='nosniff',
|
|
241
|
+
'unsafe_preview_response')
|
|
242
|
+
data=bytearray()
|
|
243
|
+
async for piece in response.aiter_bytes():
|
|
244
|
+
require(len(data)+len(piece)<=max_bytes,'preview_too_large')
|
|
245
|
+
data.extend(piece)
|
|
246
|
+
return bytes(data),{'media_type':response.headers.get('content-type','application/octet-stream'),
|
|
247
|
+
'content_disposition':response.headers.get('content-disposition'),
|
|
248
|
+
'etag':response.headers.get('etag')}
|
|
249
|
+
|
|
250
|
+
def _require_token_secret_transport(self):
|
|
251
|
+
# These envelopes contain a recovery secret. A path transport would put
|
|
252
|
+
# it in URL history and access logs, including when it compresses paths.
|
|
253
|
+
require(type(self.transport) in {HTTPTransport,GraphQLTransport,MCPHTTPTransport},
|
|
254
|
+
'secure_channel_required')
|
|
255
|
+
host=urlsplit(self.state.server).hostname
|
|
256
|
+
require(self.state.server.startswith('https://') or host in {
|
|
257
|
+
'testserver','localhost','127.0.0.1','::1'},
|
|
258
|
+
'secure_channel_required')
|
|
259
|
+
|
|
260
|
+
async def _send_token_secret(self, packet):
|
|
261
|
+
self._require_token_secret_transport()
|
|
262
|
+
return await self.send(packet)
|
|
263
|
+
|
|
264
|
+
def _token_journal(self):
|
|
265
|
+
return read_journal(self.state)
|
|
266
|
+
|
|
267
|
+
@staticmethod
|
|
268
|
+
def _token_claim(nonce,request_id,operation,subject=None):
|
|
269
|
+
if operation in {'identity.temporary','identity.custodial_create'}:
|
|
270
|
+
prefix='u_tmp_' if operation=='identity.temporary' else 'u_cust_'
|
|
271
|
+
subject=prefix+hashlib.sha256(unb64(nonce,limit=64)).hexdigest()[:32]
|
|
272
|
+
credential='t_'+subject[2:]
|
|
273
|
+
else:
|
|
274
|
+
require(subject is not None,'token_subject_required')
|
|
275
|
+
credential='t_'+digest((request_id,subject))[7:39]
|
|
276
|
+
return subject,credential
|
|
277
|
+
|
|
278
|
+
def _new_token_journal(self,path,operation,*,handle=None,contract_version=2):
|
|
279
|
+
require(self._token_journal()==(None,None),'token_operation_pending')
|
|
280
|
+
nonce=b64(os.urandom(32))
|
|
281
|
+
recovery_secret=b64(os.urandom(32))
|
|
282
|
+
request_id=uuid4().hex
|
|
283
|
+
subject,credential=self._token_claim(nonce,request_id,operation,self.state.subject)
|
|
284
|
+
pending={'server':self.state.server,'contract_version':contract_version,'operation':operation,'nonce':nonce,
|
|
285
|
+
'recovery_secret':recovery_secret,'request_id':request_id,
|
|
286
|
+
'expires_at':wire(self.clock()+timedelta(seconds=180)),
|
|
287
|
+
'subject_id':subject,'credential_id':credential}
|
|
288
|
+
if operation=='identity.temporary':
|
|
289
|
+
pending.update(public_key=b64(self.state.signer.public_key),
|
|
290
|
+
encryption_recipient=self.state.encryption_recipient)
|
|
291
|
+
if handle is not None:
|
|
292
|
+
pending['handle']=handle
|
|
293
|
+
durable_write(path,canonical(pending),mode=0o600)
|
|
294
|
+
return pending
|
|
295
|
+
|
|
296
|
+
def _existing_token_journal(self,path,operation,*,handle=None,contract_version=2):
|
|
297
|
+
existing,pending=self._token_journal()
|
|
298
|
+
if pending is None:
|
|
299
|
+
return self._new_token_journal(path,operation,handle=handle,
|
|
300
|
+
contract_version=contract_version)
|
|
301
|
+
require(existing==path and pending['operation']==operation and
|
|
302
|
+
(handle is None or pending.get('handle')==handle),
|
|
303
|
+
'token_operation_pending')
|
|
304
|
+
require(pending['contract_version']==contract_version,
|
|
305
|
+
'legacy_token_journal_requires_manual_resolution')
|
|
306
|
+
return pending
|
|
307
|
+
|
|
308
|
+
def _finish_token(self,path,pending,result):
|
|
309
|
+
if result.status=='ok':
|
|
310
|
+
require(result.data['credential_id']==pending['credential_id'] and
|
|
311
|
+
result.data['subject_id']==pending['subject_id'] and
|
|
312
|
+
bool(result.data.get('token')),'token_claim_mismatch')
|
|
313
|
+
self.state.accept_identity(result)
|
|
314
|
+
remove_journal(path)
|
|
315
|
+
return result
|
|
316
|
+
|
|
317
|
+
@staticmethod
|
|
318
|
+
def checked(result):
|
|
319
|
+
if result.status=='error':
|
|
320
|
+
raise Failure(result.error.code,result.error.field_path,retryable=result.error.retryable,
|
|
321
|
+
details=dict(result.data or {}))
|
|
322
|
+
return result
|
|
323
|
+
|
|
324
|
+
async def renew_certificate(self):
|
|
325
|
+
require(self.state.signer is not None and self.state.token is None,'signing_identity_required')
|
|
326
|
+
# Renewal must remain possible after expiry or a root trust rotation.
|
|
327
|
+
result=await self.call('identity.certificate_renew',certificates=())
|
|
328
|
+
if result.status=='ok':
|
|
329
|
+
self.state.data['certificates']=[result.data['certificate_id']]
|
|
330
|
+
self.state._save()
|
|
331
|
+
return result
|
|
332
|
+
|
|
333
|
+
async def register(self, handle):
|
|
334
|
+
require(self.state.subject is None,'identity_already_configured')
|
|
335
|
+
if self.state.pending_path.exists():
|
|
336
|
+
pending=loads(self.state.pending_path.read_bytes())
|
|
337
|
+
require(pending['handle']==handle,'registration_pending_for_other_handle')
|
|
338
|
+
else:
|
|
339
|
+
if self.state.signer is None:
|
|
340
|
+
self.state.save_signer(Ed25519Signer.generate())
|
|
341
|
+
self.state.ensure_encryption_key()
|
|
342
|
+
pending={'handle':handle,'request_id':uuid4().hex}
|
|
343
|
+
durable_write(self.state.pending_path,canonical(pending),mode=0o600)
|
|
344
|
+
signer=self.state.signer
|
|
345
|
+
recipient=self.state.ensure_encryption_key()
|
|
346
|
+
result=await self.call('identity.register',{'handle':handle,'public_key':b64(signer.public_key),
|
|
347
|
+
'encryption_recipient':recipient},contract_version=2,
|
|
348
|
+
subject=subject_id(signer.public_key),signer=signer,certificates=(),request_id=pending['request_id'])
|
|
349
|
+
if result.status=='ok':
|
|
350
|
+
self.state.accept_identity(result)
|
|
351
|
+
self.state.pending_path.unlink(missing_ok=True)
|
|
352
|
+
return result
|
|
353
|
+
|
|
354
|
+
@token_operation
|
|
355
|
+
async def temporary(self):
|
|
356
|
+
require(self.state.subject is None,'identity_already_configured')
|
|
357
|
+
self._require_token_secret_transport()
|
|
358
|
+
if self.state.signer is None:
|
|
359
|
+
self.state.save_signer(Ed25519Signer.generate())
|
|
360
|
+
signer=self.state.signer
|
|
361
|
+
recipient=self.state.ensure_encryption_key()
|
|
362
|
+
pending=self.state.directory/'temporary.json'
|
|
363
|
+
data=self._existing_token_journal(pending,'identity.temporary',contract_version=3)
|
|
364
|
+
proof=signer.sign(canonical({'subject_id':data['subject_id'],
|
|
365
|
+
'nonce':data['nonce'],'public_key':b64(signer.public_key),
|
|
366
|
+
'encryption_recipient':recipient,'request_id':data['request_id']}),
|
|
367
|
+
purpose='temporary-key-possession-v1')
|
|
368
|
+
packet=self.prepare('identity.temporary',{'nonce':data['nonce'],
|
|
369
|
+
'recovery_secret':data['recovery_secret'],
|
|
370
|
+
'public_key':b64(signer.public_key),'encryption_recipient':recipient,
|
|
371
|
+
'possession_proof':wire(proof)},request_id=data['request_id'],
|
|
372
|
+
anonymous=True,contract_version=3,expires_at=self.clock()+timedelta(seconds=180))
|
|
373
|
+
return self._finish_token(pending,data,await self._send_token_secret(packet))
|
|
374
|
+
|
|
375
|
+
@token_operation
|
|
376
|
+
async def custodial(self,handle):
|
|
377
|
+
require(self.state.subject is None and self.state.signer is None and
|
|
378
|
+
self.state.encryption_recipient is None,'identity_already_configured')
|
|
379
|
+
self._require_token_secret_transport()
|
|
380
|
+
pending=self.state.directory/'custodial-bootstrap.json'
|
|
381
|
+
data=self._existing_token_journal(pending,'identity.custodial_create',handle=handle)
|
|
382
|
+
packet=self.prepare('identity.custodial_create',{'handle':handle,'nonce':data['nonce'],
|
|
383
|
+
'recovery_secret':data['recovery_secret']},request_id=data['request_id'],
|
|
384
|
+
anonymous=True,contract_version=2,expires_at=self.clock()+timedelta(seconds=180))
|
|
385
|
+
return self._finish_token(pending,data,await self._send_token_secret(packet))
|
|
386
|
+
|
|
387
|
+
@token_operation
|
|
388
|
+
async def rotate_token(self):
|
|
389
|
+
require(self.state.token is not None,'token_required')
|
|
390
|
+
self._require_token_secret_transport()
|
|
391
|
+
pending=self.state.directory/'token-rotation.json'
|
|
392
|
+
data=self._existing_token_journal(pending,'identity.token_rotate')
|
|
393
|
+
packet=self.prepare('identity.token_rotate',{'nonce':data['nonce'],
|
|
394
|
+
'recovery_secret':data['recovery_secret']},request_id=data['request_id'],
|
|
395
|
+
contract_version=2,expires_at=self.clock()+timedelta(seconds=180))
|
|
396
|
+
return self._finish_token(pending,data,await self._send_token_secret(packet))
|
|
397
|
+
|
|
398
|
+
@token_operation
|
|
399
|
+
async def recover_token(self):
|
|
400
|
+
self._require_token_secret_transport()
|
|
401
|
+
path,pending=self._token_journal()
|
|
402
|
+
require(path is not None,'token_recovery_not_pending')
|
|
403
|
+
# A crash after writing client.json but before unlinking the journal is
|
|
404
|
+
# complete locally. Never revoke that already accepted credential.
|
|
405
|
+
accepted={pending['credential_id'], (pending.get('recovery') or {}).get('credential_id')}
|
|
406
|
+
if self.state.token and self.state.token[0] in accepted:
|
|
407
|
+
remove_journal(path)
|
|
408
|
+
raise Failure('token_recovery_not_pending')
|
|
409
|
+
recovery=pending.get('recovery')
|
|
410
|
+
if recovery is None:
|
|
411
|
+
nonce=b64(os.urandom(32))
|
|
412
|
+
request_id=uuid4().hex
|
|
413
|
+
_,credential=self._token_claim(nonce,request_id,'identity.token_recover',
|
|
414
|
+
pending['subject_id'])
|
|
415
|
+
recovery={'nonce':nonce,'new_recovery_secret':b64(os.urandom(32)),
|
|
416
|
+
'request_id':request_id,'credential_id':credential,
|
|
417
|
+
'expires_at':wire(self.clock()+timedelta(seconds=180))}
|
|
418
|
+
pending['recovery']=recovery
|
|
419
|
+
durable_write(path,canonical(pending),mode=0o600)
|
|
420
|
+
args={'credential_id':pending['credential_id'],
|
|
421
|
+
'original_request_id':pending['request_id'],
|
|
422
|
+
'recovery_secret':pending['recovery_secret'],
|
|
423
|
+
'nonce':recovery['nonce'],
|
|
424
|
+
'new_recovery_secret':recovery['new_recovery_secret']}
|
|
425
|
+
packet=request_for('identity.token_recover',args,self.state.server,
|
|
426
|
+
subject=pending['subject_id'],request_id=recovery['request_id'],
|
|
427
|
+
expires_at=self.clock()+timedelta(seconds=180),source='msg')
|
|
428
|
+
result=await self._send_token_secret(packet)
|
|
429
|
+
if result.status=='ok':
|
|
430
|
+
require(result.data['credential_id']==recovery['credential_id'] and
|
|
431
|
+
result.data['subject_id']==pending['subject_id'] and
|
|
432
|
+
result.data['previous_credential']==pending['credential_id'] and
|
|
433
|
+
bool(result.data.get('token')),'token_claim_mismatch')
|
|
434
|
+
self.state.accept_identity(result)
|
|
435
|
+
remove_journal(path)
|
|
436
|
+
elif result.error.code=='token_delivery_unavailable':
|
|
437
|
+
# Claim committed but its response may have vanished. A further
|
|
438
|
+
# explicit recovery consumes the newly bound secret, never @1.
|
|
439
|
+
pending.update(credential_id=recovery['credential_id'],
|
|
440
|
+
request_id=recovery['request_id'],
|
|
441
|
+
recovery_secret=recovery['new_recovery_secret'])
|
|
442
|
+
pending.pop('recovery')
|
|
443
|
+
durable_write(path,canonical(pending),mode=0o600)
|
|
444
|
+
return result
|
|
445
|
+
|
|
446
|
+
async def upgrade(self,handle=None):
|
|
447
|
+
from msg.client_upgrade import upgrade_identity
|
|
448
|
+
return await upgrade_identity(self,handle)
|
|
449
|
+
|
|
450
|
+
async def upgrade_custodial(self,handle, *, external_ciphertexts_migrated: bool):
|
|
451
|
+
from msg.client_upgrade import locked_state
|
|
452
|
+
self._require_token_secret_transport()
|
|
453
|
+
with locked_state(self.state):
|
|
454
|
+
require(not any((self.state.directory/name).exists() or
|
|
455
|
+
(self.state.directory/name).is_symlink() for name in
|
|
456
|
+
('identity-upgrade.json','temporary.json',
|
|
457
|
+
'custodial-bootstrap.json','token-rotation.json')),
|
|
458
|
+
'identity_recovery_pending')
|
|
459
|
+
return await self._upgrade_custodial(handle,
|
|
460
|
+
external_ciphertexts_migrated=external_ciphertexts_migrated)
|
|
461
|
+
|
|
462
|
+
async def _upgrade_custodial(self,handle, *, external_ciphertexts_migrated: bool):
|
|
463
|
+
require(type(external_ciphertexts_migrated) is bool,'migration_statement_required')
|
|
464
|
+
require(self.state.subject is not None and self.state.token is not None,
|
|
465
|
+
'custodial_token_required')
|
|
466
|
+
journal=self.state.directory/'custodial-upgrade.json'
|
|
467
|
+
if self.state.signer is None:
|
|
468
|
+
self.state.save_signer(Ed25519Signer.generate())
|
|
469
|
+
recipient=self.state.ensure_encryption_key()
|
|
470
|
+
public=b64(self.state.signer.public_key)
|
|
471
|
+
if journal.exists():
|
|
472
|
+
pending=loads(journal.read_bytes())
|
|
473
|
+
require(pending['handle']==handle and pending['public_key']==public and
|
|
474
|
+
pending['encryption_recipient']==recipient,'custodial_upgrade_journal_mismatch')
|
|
475
|
+
else:
|
|
476
|
+
pending={'handle':handle,'public_key':public,'encryption_recipient':recipient,
|
|
477
|
+
'server':self.state.server,'subject_id':self.state.subject,
|
|
478
|
+
'start_request_id':uuid4().hex,'finish_request_id':uuid4().hex,
|
|
479
|
+
'challenge':None}
|
|
480
|
+
durable_write(journal,canonical(pending),mode=0o600)
|
|
481
|
+
if pending['challenge'] is None:
|
|
482
|
+
signed={'subject_id':self.state.subject,'handle':handle,'public_key':public,
|
|
483
|
+
'encryption_recipient':recipient,'request_id':pending['start_request_id']}
|
|
484
|
+
proof=self.state.signer.sign(canonical(signed),purpose='custodial-upgrade-start')
|
|
485
|
+
started=await self.call('identity.custodial_upgrade_start',{
|
|
486
|
+
'handle':handle,'public_key':public,'encryption_recipient':recipient,
|
|
487
|
+
'possession_proof':wire(proof)},request_id=pending['start_request_id'])
|
|
488
|
+
if started.status!='ok':
|
|
489
|
+
return started
|
|
490
|
+
pending['challenge']=dict(started.data)
|
|
491
|
+
durable_write(journal,canonical(pending),mode=0o600)
|
|
492
|
+
challenge=pending['challenge']
|
|
493
|
+
if pending.get('status')=='pending_rewrap':
|
|
494
|
+
return await self.call('identity.custodial_upgrade_inventory',
|
|
495
|
+
{'challenge_id':challenge['challenge_id']})
|
|
496
|
+
age_identity=self.state.age_key_path.read_text().strip()
|
|
497
|
+
require(recipient_from_identity(age_identity)==recipient,'encryption_identity_mismatch')
|
|
498
|
+
age_proof=client_upgrade_proof(age_identity,challenge)
|
|
499
|
+
acknowledgement={'subject_id':self.state.subject,'challenge_id':challenge['challenge_id'],
|
|
500
|
+
'age_proof':age_proof,'external_ciphertexts_migrated':external_ciphertexts_migrated,
|
|
501
|
+
'request_id':pending['finish_request_id']}
|
|
502
|
+
signed_ack=self.state.signer.sign(canonical(acknowledgement),
|
|
503
|
+
purpose='custodial-upgrade-finish')
|
|
504
|
+
async def recover_with_new_key():
|
|
505
|
+
return await self.call('identity.custodial_upgrade_result',
|
|
506
|
+
{'challenge_id':challenge['challenge_id']},signer=self.state.signer,
|
|
507
|
+
subject=self.state.subject,certificates=())
|
|
508
|
+
try:
|
|
509
|
+
result=await self.call('identity.custodial_upgrade_finish',{
|
|
510
|
+
'challenge_id':challenge['challenge_id'],'age_proof':age_proof,
|
|
511
|
+
'external_ciphertexts_migrated':external_ciphertexts_migrated,
|
|
512
|
+
'migration_ack':wire(signed_ack)},request_id=pending['finish_request_id'])
|
|
513
|
+
except Failure as exc:
|
|
514
|
+
if exc.code!='transport_uncertain':
|
|
515
|
+
raise
|
|
516
|
+
recovered=await recover_with_new_key()
|
|
517
|
+
if recovered.status!='ok':
|
|
518
|
+
raise
|
|
519
|
+
result=recovered
|
|
520
|
+
if result.status=='error' and result.error.code in {
|
|
521
|
+
'credential_expired','credential_revoked','request_expired','idempotency_conflict'}:
|
|
522
|
+
recovered=await recover_with_new_key()
|
|
523
|
+
if recovered.status=='ok':
|
|
524
|
+
result=recovered
|
|
525
|
+
if result.status=='ok' and result.data['status']=='completed':
|
|
526
|
+
self.state.accept_identity(result)
|
|
527
|
+
journal.unlink(missing_ok=True)
|
|
528
|
+
elif result.status=='ok' and result.data['status']=='pending_rewrap':
|
|
529
|
+
# Preserve the bound challenge and new private keys for the
|
|
530
|
+
# per-entry rewrap inventory and signed client acknowledgements.
|
|
531
|
+
# A later finish attempt has its own request ID; replaying this
|
|
532
|
+
# one would only retrieve the original pending result.
|
|
533
|
+
pending['status']='pending_rewrap'
|
|
534
|
+
pending['finish_request_id']=uuid4().hex
|
|
535
|
+
durable_write(journal,canonical(pending),mode=0o600)
|
|
536
|
+
return result
|
|
537
|
+
|
|
538
|
+
async def rotate_encryption_key(self):
|
|
539
|
+
require(self.state.subject is not None and self.state.signer is not None and
|
|
540
|
+
self.state.token is None,'signing_identity_required')
|
|
541
|
+
journal=self.state.directory/'encryption-rotation.json'
|
|
542
|
+
pending_key=self.state.directory/'encryption.pending.agekey'
|
|
543
|
+
if journal.exists():
|
|
544
|
+
pending=loads(journal.read_bytes())
|
|
545
|
+
require(pending_key.is_file() and not pending_key.is_symlink() and
|
|
546
|
+
pending_key.stat().st_uid==os.geteuid() and pending_key.stat().st_mode&0o077==0,
|
|
547
|
+
'unsafe_client_key_permissions')
|
|
548
|
+
require(recipient_from_identity(pending_key.read_text().strip())==pending['recipient'],
|
|
549
|
+
'encryption_rotation_mismatch')
|
|
550
|
+
else:
|
|
551
|
+
require(self.state.encryption_recipient is not None,'encryption_key_not_found')
|
|
552
|
+
identity,recipient=generate_age_key()
|
|
553
|
+
durable_write(pending_key,(identity+'\n').encode(),mode=0o600)
|
|
554
|
+
pending={'recipient':recipient,'request_id':uuid4().hex}
|
|
555
|
+
durable_write(journal,canonical(pending),mode=0o600)
|
|
556
|
+
result=await self.call('identity.encryption_key_rotate',
|
|
557
|
+
{'encryption_recipient':pending['recipient']},
|
|
558
|
+
request_id=pending['request_id'])
|
|
559
|
+
if result.status=='ok':
|
|
560
|
+
require(result.data['recipient']==pending['recipient'],
|
|
561
|
+
'encryption_rotation_mismatch')
|
|
562
|
+
previous=result.data['previous_key_id']
|
|
563
|
+
history=self.state.directory/('encryption-'+previous+'.agekey')
|
|
564
|
+
if self.state.age_key_path.exists() and self.state.encryption_recipient!=pending['recipient']:
|
|
565
|
+
require(not history.exists(),'encryption_history_conflict')
|
|
566
|
+
os.replace(self.state.age_key_path,history)
|
|
567
|
+
if not self.state.age_key_path.exists():
|
|
568
|
+
os.replace(pending_key,self.state.age_key_path)
|
|
569
|
+
self.state.encryption_recipient=pending['recipient']
|
|
570
|
+
pending_key.unlink(missing_ok=True)
|
|
571
|
+
journal.unlink(missing_ok=True)
|
|
572
|
+
return result
|
|
573
|
+
|
|
574
|
+
async def upload(self, path, *, transfer_id=None, part_bytes=65536, media_type='application/octet-stream',target=None):
|
|
575
|
+
path=Path(path)
|
|
576
|
+
size, hashed=await asyncio.to_thread(hash_file,path)
|
|
577
|
+
journal=self.state.directory/('upload-'+digest((str(path.resolve()),hashed))[7:39]+'.json')
|
|
578
|
+
saved=loads(journal.read_bytes()) if journal.exists() else None
|
|
579
|
+
if transfer_id is None and saved:
|
|
580
|
+
transfer_id=saved.get('transfer_id')
|
|
581
|
+
if transfer_id is None:
|
|
582
|
+
limits=await self.transport.discover()
|
|
583
|
+
arguments={'direction':'upload','size':size,'digest':hashed,'media_type':media_type,
|
|
584
|
+
'requested_part_bytes':part_bytes,'max_request_bytes':limits.max_request_bytes,
|
|
585
|
+
'max_response_bytes':limits.max_response_bytes,'max_path_bytes':limits.max_path_bytes}
|
|
586
|
+
if target:
|
|
587
|
+
arguments['target']=wire(target if isinstance(target,ResourceRef) else ResourceRef(id=target))
|
|
588
|
+
saved=saved or {'request_id':uuid4().hex,'size':size,'digest':hashed}
|
|
589
|
+
durable_write(journal,canonical(saved),mode=0o600)
|
|
590
|
+
result=self.checked(await self.call('transfer.open',arguments,request_id=saved['request_id']))
|
|
591
|
+
transfer_id=result.data['transfer_id']
|
|
592
|
+
part_bytes=result.data['part_bytes']
|
|
593
|
+
saved.update(transfer_id=transfer_id,part_bytes=part_bytes)
|
|
594
|
+
durable_write(journal,canonical(saved),mode=0o600)
|
|
595
|
+
elif saved and saved.get('transfer_id')==transfer_id:
|
|
596
|
+
part_bytes=min(part_bytes,saved['part_bytes'])
|
|
597
|
+
with path.open('rb') as stream:
|
|
598
|
+
cursor=None
|
|
599
|
+
while True:
|
|
600
|
+
args={'transfer_id':transfer_id,'limit':50}
|
|
601
|
+
if cursor: args['cursor']=cursor
|
|
602
|
+
status=self.checked(await self.call('transfer.status',args))
|
|
603
|
+
require(status.data['size']==size and status.data['digest']==hashed,'transfer_source_mismatch')
|
|
604
|
+
if status.data['state']=='sealed':
|
|
605
|
+
break
|
|
606
|
+
require(status.data['state']=='open','transfer_not_open')
|
|
607
|
+
for start,end in status.data['missing']:
|
|
608
|
+
stream.seek(start)
|
|
609
|
+
while start<end:
|
|
610
|
+
piece=stream.read(min(part_bytes,end-start))
|
|
611
|
+
require(bool(piece),'source_changed_during_upload')
|
|
612
|
+
self.checked(await self.call('transfer.part_put',{'transfer_id':transfer_id,
|
|
613
|
+
'offset':start,'data':b64(piece),'digest':digest(piece)}))
|
|
614
|
+
start+=len(piece)
|
|
615
|
+
cursor=status.data.get('next_cursor')
|
|
616
|
+
if cursor is None: break
|
|
617
|
+
result=self.checked(await self.call('transfer.seal',{'transfer_id':transfer_id,'final_size':size,'final_digest':hashed}))
|
|
618
|
+
journal.unlink(missing_ok=True)
|
|
619
|
+
return result
|
|
620
|
+
|
|
621
|
+
async def download(self, resource, path, *, transfer_id=None, part_bytes=65536):
|
|
622
|
+
ref=resource if isinstance(resource,ResourceRef) else ResourceRef(id=resource)
|
|
623
|
+
path=Path(path)
|
|
624
|
+
journal=self.state.directory/('download-'+digest((wire(ref),str(path.resolve())))[7:39]+'.json')
|
|
625
|
+
part=path.with_name(path.name+'.msg-part')
|
|
626
|
+
saved=loads(journal.read_bytes()) if journal.exists() else None
|
|
627
|
+
if transfer_id is None and saved:
|
|
628
|
+
transfer_id=saved['transfer_id']
|
|
629
|
+
if transfer_id is None:
|
|
630
|
+
limits=await self.transport.discover()
|
|
631
|
+
result=self.checked(await self.call('transfer.open',{'direction':'download','target':wire(ref),
|
|
632
|
+
'requested_part_bytes':part_bytes,'max_request_bytes':limits.max_request_bytes,
|
|
633
|
+
'max_response_bytes':limits.max_response_bytes,'max_path_bytes':limits.max_path_bytes}))
|
|
634
|
+
saved={'transfer_id':result.data['transfer_id'],'size':result.data['size'],'digest':result.data['digest'],
|
|
635
|
+
'part_bytes':result.data['part_bytes'],'offset':0,'prefix_digest':digest(b'')}
|
|
636
|
+
transfer_id=saved['transfer_id']
|
|
637
|
+
durable_write(journal,canonical(saved),mode=0o600)
|
|
638
|
+
require(saved is not None,'download_journal_required')
|
|
639
|
+
require(not path.exists() and not part.is_symlink(),'download_target_exists')
|
|
640
|
+
path.parent.mkdir(parents=True,exist_ok=True)
|
|
641
|
+
offset=saved['offset']
|
|
642
|
+
hasher=hashlib.sha256()
|
|
643
|
+
if offset:
|
|
644
|
+
require(part.is_file() and part.stat().st_size>=offset,'download_partial_missing')
|
|
645
|
+
with part.open('rb') as source:
|
|
646
|
+
remaining=offset
|
|
647
|
+
while remaining:
|
|
648
|
+
piece=source.read(min(65536,remaining));require(bool(piece),'download_partial_missing')
|
|
649
|
+
hasher.update(piece);remaining-=len(piece)
|
|
650
|
+
require('sha256:'+hasher.hexdigest()==saved['prefix_digest'],'download_partial_modified')
|
|
651
|
+
mode='r+b' if part.exists() else 'w+b'
|
|
652
|
+
with part.open(mode) as out:
|
|
653
|
+
part.chmod(0o600);out.truncate(offset);out.seek(offset)
|
|
654
|
+
while offset<saved['size']:
|
|
655
|
+
amount=min(part_bytes,saved['part_bytes'],saved['size']-offset)
|
|
656
|
+
result=self.checked(await self.call('transfer.part_get',{'transfer_id':transfer_id,'offset':offset,'length':amount}))
|
|
657
|
+
piece=unb64(result.data['data'])
|
|
658
|
+
require(len(piece)==amount and digest(piece)==result.data['chunk']['content']['digest'],'download_chunk_digest_mismatch')
|
|
659
|
+
out.write(piece);out.flush();os.fsync(out.fileno())
|
|
660
|
+
hasher.update(piece);offset+=len(piece)
|
|
661
|
+
saved.update(offset=offset,prefix_digest='sha256:'+hasher.hexdigest())
|
|
662
|
+
durable_write(journal,canonical(saved),mode=0o600)
|
|
663
|
+
require('sha256:'+hasher.hexdigest()==saved['digest'],'download_digest_mismatch')
|
|
664
|
+
# Link avoids overwriting a target created concurrently.
|
|
665
|
+
os.link(part,path);part.unlink();journal.unlink(missing_ok=True)
|
|
666
|
+
return {'path':str(path),'size':saved['size'],'digest':saved['digest'],'transfer_id':transfer_id}
|
|
667
|
+
|
|
668
|
+
async def ack(self,ref):
|
|
669
|
+
require(ref.revision is not None,'ack_revision_required')
|
|
670
|
+
value=self.checked(await self.call('discovery.get',{'id':ref.id,'revision':ref.revision,
|
|
671
|
+
'fields':['id','revision','digest']}))
|
|
672
|
+
return await self.call('discussion.ack',{'target':wire(ref),'digest':value.data['digest']})
|
|
673
|
+
|
|
674
|
+
async def contract_registry(self):
|
|
675
|
+
from types import SimpleNamespace
|
|
676
|
+
from msg.core.cursors import CursorCodec
|
|
677
|
+
result=self.checked(await self.call('discovery.operations',{},anonymous=True))
|
|
678
|
+
return RemoteRegistry(self,result.data)
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
class RemoteRegistry:
|
|
682
|
+
"""Read-only contract cache; never loads code or trusts server principals."""
|
|
683
|
+
def __init__(self,client,catalog):
|
|
684
|
+
from types import SimpleNamespace
|
|
685
|
+
self.client=client
|
|
686
|
+
self._catalog=dict(catalog)
|
|
687
|
+
self._specs={}
|
|
688
|
+
self._schemas={}
|
|
689
|
+
self.directory=client.state.directory/'cache'/catalog['digest'].replace(':','-')
|
|
690
|
+
self.directory.mkdir(parents=True,exist_ok=True)
|
|
691
|
+
for item in catalog['operations']:
|
|
692
|
+
key=(item['name'],item['version'])
|
|
693
|
+
require(key not in self._specs,'duplicate_operation')
|
|
694
|
+
require('network' in item['entries'] and not item['name'].startswith('root.'),'invalid_network_contract')
|
|
695
|
+
self._specs[key]=SimpleNamespace(**{**item,'entries':frozenset(item['entries']),
|
|
696
|
+
'input_schema':decode(ResourceRef,item['input_schema']),'output_schema':decode(ResourceRef,item['output_schema'])})
|
|
697
|
+
|
|
698
|
+
def catalog(self):
|
|
699
|
+
return self._catalog
|
|
700
|
+
|
|
701
|
+
def operations(self,entry='network'):
|
|
702
|
+
return tuple(self._specs[key] for key in sorted(self._specs) if entry in self._specs[key].entries)
|
|
703
|
+
|
|
704
|
+
def operation(self,name,version=1):
|
|
705
|
+
require((name,version) in self._specs,'unknown_operation')
|
|
706
|
+
return self._specs[(name,version)]
|
|
707
|
+
|
|
708
|
+
def schema(self,ref):
|
|
709
|
+
require(ref.id in self._schemas,'schema_not_cached')
|
|
710
|
+
return self._schemas[ref.id]
|
|
711
|
+
|
|
712
|
+
async def load_schemas(self,specs):
|
|
713
|
+
async def load(spec):
|
|
714
|
+
path=self.directory/(digest((spec.name,spec.version))[7:39]+'.json')
|
|
715
|
+
if path.is_file():
|
|
716
|
+
value=loads(path.read_bytes())
|
|
717
|
+
else:
|
|
718
|
+
identity=spec.name if spec.version==1 else f'{spec.name}@{spec.version}'
|
|
719
|
+
result=self.client.checked(await self.client.call('discovery.schema',{'operation':identity},anonymous=True))
|
|
720
|
+
value=dict(result.data)
|
|
721
|
+
durable_write(path,canonical(value),mode=0o600)
|
|
722
|
+
require(value['operation']['name']==spec.name and value['operation']['version']==spec.version,'schema_contract_mismatch')
|
|
723
|
+
self._schemas[spec.input_schema.id]=value['input']
|
|
724
|
+
self._schemas[spec.output_schema.id]=value['output']
|
|
725
|
+
await asyncio.gather(*(load(s) for s in specs if s.input_schema.id not in self._schemas))
|