msgctl 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- msg/__init__.py +2 -0
- msg/admin/__init__.py +0 -0
- msg/admin/backup_retirement.py +56 -0
- msg/admin/backups.py +341 -0
- msg/admin/custodial_check.py +77 -0
- msg/admin/diagnostics.py +750 -0
- msg/admin/market.py +91 -0
- msg/admin/market_check.py +341 -0
- msg/admin/money.py +322 -0
- msg/admin/preflight.py +142 -0
- msg/admin/recovery_replay.py +281 -0
- msg/admin/restore_database.py +37 -0
- msg/admin/root.py +347 -0
- msg/admin/rotation.py +197 -0
- msg/admin/token_delivery_check.py +54 -0
- msg/admin/upgrade_check.py +58 -0
- msg/application.py +163 -0
- msg/bootstrap.py +305 -0
- msg/cli.py +537 -0
- msg/client.py +725 -0
- msg/client_certificates.py +98 -0
- msg/client_content.py +43 -0
- msg/client_custodial.py +206 -0
- msg/client_market.py +104 -0
- msg/client_recovery.py +186 -0
- msg/client_secrets.py +58 -0
- msg/client_tokens.py +99 -0
- msg/client_upgrade.py +188 -0
- msg/config.py +318 -0
- msg/constants.py +11 -0
- msg/core/__init__.py +0 -0
- msg/core/batching.py +23 -0
- msg/core/codec.py +253 -0
- msg/core/contracts.py +128 -0
- msg/core/cursors.py +68 -0
- msg/core/email_address.py +33 -0
- msg/core/errors.py +22 -0
- msg/core/events.py +6 -0
- msg/core/execution_ports.py +20 -0
- msg/core/executor.py +193 -0
- msg/core/models.py +517 -0
- msg/core/packet.py +71 -0
- msg/core/permissions.py +14 -0
- msg/core/query.py +44 -0
- msg/core/read_query.py +60 -0
- msg/core/registry.py +173 -0
- msg/core/requests.py +58 -0
- msg/core/schema_policy.py +28 -0
- msg/core/schemas.py +30 -0
- msg/core/tags.py +21 -0
- msg/core/template_dsl.py +94 -0
- msg/core/text_patch.py +283 -0
- msg/core/tool_execution.py +28 -0
- msg/core/transfer.py +183 -0
- msg/daemon.py +255 -0
- msg/data/__init__.py +0 -0
- msg/data/bootstrap.json +72 -0
- msg/data/favicon.png +0 -0
- msg/data/logo-dark.svg +5 -0
- msg/data/logo.svg +5 -0
- msg/data/recovery-checkpoint.example.json +1 -0
- msg/data/recovery-checkpoint.schema.json +118 -0
- msg/data/shortcodes.json +1 -0
- msg/data/system/AGENTS.md +6 -0
- msg/data/system/rules/_index.md +17 -0
- msg/data/system/rules/auth.md +6 -0
- msg/data/system/rules/files.md +6 -0
- msg/data/system/rules/identity.md +6 -0
- msg/data/system/rules/protocol.md +6 -0
- msg/data/system/rules/read-write.md +6 -0
- msg/data/system/rules/recovery.md +6 -0
- msg/data/system/rules/security.md +10 -0
- msg/data/system/rules/topics.md +6 -0
- msg/extensions/__init__.py +1 -0
- msg/extensions/hosting.py +259 -0
- msg/extensions/keystore.py +96 -0
- msg/extensions/repositories.py +769 -0
- msg/extensions/rss.py +67 -0
- msg/extensions/ssh.py +264 -0
- msg/extensions/ssh_git.py +300 -0
- msg/extensions/tools.py +150 -0
- msg/hosting_runtime.py +138 -0
- msg/market/__init__.py +1 -0
- msg/market/arbitration.py +481 -0
- msg/market/delivery.py +338 -0
- msg/market/delivery_notifications.py +110 -0
- msg/market/delivery_targets.py +76 -0
- msg/market/email.py +100 -0
- msg/market/escrow.py +345 -0
- msg/market/orders.py +218 -0
- msg/market/policy.py +147 -0
- msg/market/rationale.py +91 -0
- msg/market/references.py +19 -0
- msg/market/targets.py +157 -0
- msg/plugins/__init__.py +28 -0
- msg/plugins/achievements.py +316 -0
- msg/plugins/batch.py +32 -0
- msg/plugins/bounty.py +366 -0
- msg/plugins/collaboration.py +261 -0
- msg/plugins/common.py +217 -0
- msg/plugins/communication.py +853 -0
- msg/plugins/content.py +855 -0
- msg/plugins/custodial_lifecycle.py +224 -0
- msg/plugins/delivery.py +230 -0
- msg/plugins/discovery.py +1266 -0
- msg/plugins/discussion.py +162 -0
- msg/plugins/extensions.py +10 -0
- msg/plugins/following.py +78 -0
- msg/plugins/hosting_capacity.py +52 -0
- msg/plugins/identity.py +1654 -0
- msg/plugins/money.py +221 -0
- msg/plugins/offers.py +183 -0
- msg/plugins/orders.py +273 -0
- msg/plugins/recovery.py +376 -0
- msg/plugins/schemas.py +5 -0
- msg/plugins/sharing.py +300 -0
- msg/plugins/store.py +249 -0
- msg/plugins/system.py +75 -0
- msg/plugins/transfer.py +249 -0
- msg/py.typed +0 -0
- msg/security/__init__.py +0 -0
- msg/security/age_keys.py +112 -0
- msg/security/authentication.py +141 -0
- msg/security/authorization.py +300 -0
- msg/security/backup_retirement.py +135 -0
- msg/security/capabilities.py +157 -0
- msg/security/certificates.py +183 -0
- msg/security/crypto.py +102 -0
- msg/security/custodial_migration.py +292 -0
- msg/security/custody_history.py +146 -0
- msg/security/network.py +81 -0
- msg/security/policy.py +67 -0
- msg/security/quarantine.py +19 -0
- msg/security/root_files.py +67 -0
- msg/security/rotation_journal.py +60 -0
- msg/security/sealed_box.py +70 -0
- msg/security/sharing_policy.py +29 -0
- msg/security/token_delivery.py +93 -0
- msg/security/vault.py +186 -0
- msg/storage/__init__.py +0 -0
- msg/storage/capacity.py +100 -0
- msg/storage/custodial_migration.py +22 -0
- msg/storage/git.py +435 -0
- msg/storage/ledger_migration.py +307 -0
- msg/storage/market_migration.py +49 -0
- msg/storage/postgres.py +663 -0
- msg/storage/query.py +39 -0
- msg/storage/read_only.py +126 -0
- msg/storage/session.py +403 -0
- msg/storage/sqlite.py +360 -0
- msg/storage/topic_event_migration.py +26 -0
- msg/storage/valkey_bus.py +45 -0
- msg/transports/__init__.py +0 -0
- msg/transports/client.py +183 -0
- msg/transports/dictionary.py +387 -0
- msg/transports/graphql.py +75 -0
- msg/transports/http.py +72 -0
- msg/transports/http_routes.py +1577 -0
- msg/transports/mcp.py +127 -0
- msg/transports/packet.py +42 -0
- msg/transports/read_tree_path.py +65 -0
- msg/transports/stdio.py +36 -0
- msg/transports/url_safety.py +123 -0
- msg/tui.py +368 -0
- msg/workers/__init__.py +1 -0
- msg/workers/effects.py +406 -0
- msg/workers/leases.py +19 -0
- msg/workers/mail.py +63 -0
- msg/workers/maintenance.py +310 -0
- msg/workers/sandbox.py +74 -0
- msg/workers/sandbox_child.py +189 -0
- msg/workers/webhook.py +160 -0
- msgctl-0.1.0a1.dist-info/METADATA +96 -0
- msgctl-0.1.0a1.dist-info/RECORD +176 -0
- msgctl-0.1.0a1.dist-info/WHEEL +4 -0
- msgctl-0.1.0a1.dist-info/entry_points.txt +4 -0
msg/plugins/transfer.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
"""All network adapters expose these same six transfer operations."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
from datetime import timedelta
|
|
5
|
+
from types import SimpleNamespace
|
|
6
|
+
|
|
7
|
+
from msg.core.codec import b64,unb64,wire,decode,canonical,loads,parse_time,digest
|
|
8
|
+
from msg.core.errors import Failure,require
|
|
9
|
+
from msg.core.models import ResourceRef,HandlerOutput
|
|
10
|
+
from msg.core.tags import normalize_tag
|
|
11
|
+
from msg.core.read_query import read_query_version
|
|
12
|
+
from msg.core.transfer import TransferService
|
|
13
|
+
from msg.plugins.common import registration,resolve,create_resource,check_access
|
|
14
|
+
from msg.plugins.schemas import obj,INTEGER,STRING,IDENTIFIER,BYTES,REF
|
|
15
|
+
|
|
16
|
+
DIGEST={'type':'string','pattern':'^sha256:[0-9a-f]{64}$'}
|
|
17
|
+
SIZE={'type':'integer','minimum':0,'maximum':2**63-1}
|
|
18
|
+
QUERY_MEDIA='application/vnd.msg.read-query+json'
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def sealed_read_query(app,ctx,request,tx,transfer):
|
|
22
|
+
require(transfer.subject_id==ctx.principal.subject,'query_ref_principal_mismatch')
|
|
23
|
+
require(transfer.state=='sealed' and transfer.direction=='upload' and
|
|
24
|
+
transfer.output is not None,'query_transfer_not_sealed')
|
|
25
|
+
require(transfer.expires_at>ctx.now,'query_ref_expired')
|
|
26
|
+
ref=transfer.output
|
|
27
|
+
resource=await tx.resource(ref.id)
|
|
28
|
+
require(resource.type=='file' and resource.owner==ctx.principal.subject and
|
|
29
|
+
resource.state=='active','invalid_query_ref')
|
|
30
|
+
await check_access(app,ctx,request,tx,ref.id,'read')
|
|
31
|
+
revision=await tx.revision(ref)
|
|
32
|
+
require(revision.content.media_type==QUERY_MEDIA and revision.content.size<=65536 and
|
|
33
|
+
revision.content.digest==transfer.expected_digest,'query_ref_digest_mismatch')
|
|
34
|
+
try:
|
|
35
|
+
descriptor=loads(await app.contents.read_bytes(revision.content,limit=65536))
|
|
36
|
+
except Failure as exc:
|
|
37
|
+
raise Failure('invalid_query_ref') from exc
|
|
38
|
+
require(type(descriptor) is dict and set(descriptor)=={'version','kind','arguments'} and
|
|
39
|
+
descriptor['version']==1 and descriptor['kind'] in {'read','search'} and
|
|
40
|
+
type(descriptor['arguments']) is dict,'invalid_query_ref')
|
|
41
|
+
args=descriptor['arguments']
|
|
42
|
+
kind=descriptor['kind']
|
|
43
|
+
require(args.get('parent' if kind=='read' else 'scope') is not None and
|
|
44
|
+
'cursor' not in args,'invalid_query_ref')
|
|
45
|
+
operation='discovery.read_query' if kind=='read' else 'discovery.lexical_search'
|
|
46
|
+
contract_version=(read_query_version(args) if kind=='read' else
|
|
47
|
+
4 if kind=='search' and 'suggest' in args else
|
|
48
|
+
3 if kind=='search' and any(name in args for name in
|
|
49
|
+
('source_kind','relation_type')) else 2 if kind=='search' and 'facets' in args else 1)
|
|
50
|
+
try:
|
|
51
|
+
app.registry.validate(app.registry.operation(operation,contract_version).input_schema,args)
|
|
52
|
+
except Failure as exc:
|
|
53
|
+
raise Failure('invalid_query_ref') from exc
|
|
54
|
+
return ref,revision,kind,args
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def query_ref_principal(principal):
|
|
58
|
+
return {'actor':principal.actor,'subject':principal.subject,
|
|
59
|
+
'credential_id':principal.credential_id}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def negotiate(app,request):
|
|
63
|
+
a=request.arguments
|
|
64
|
+
limits=app.settings.server.limits
|
|
65
|
+
request_bytes=min(limits.max_request_bytes,a.get('max_request_bytes',limits.max_request_bytes))
|
|
66
|
+
response_bytes=min(limits.max_response_bytes,a.get('max_response_bytes',limits.max_response_bytes))
|
|
67
|
+
path_bytes=min(limits.max_path_bytes,a.get('max_path_bytes',limits.max_path_bytes))
|
|
68
|
+
# Calculate an envelope with maximal fixed-length IDs/offsets and the caller's
|
|
69
|
+
# actual proof overhead. Two Base64 layers are needed for data inside a path.
|
|
70
|
+
packet=wire(request)
|
|
71
|
+
packet.update(operation='transfer.part_put',request_id='r'*64,payload_digest='sha256:'+'0'*64,
|
|
72
|
+
arguments={'transfer_id':'tr_'+'0'*32,'offset':2**63-1,'data':'','digest':'sha256:'+'0'*64})
|
|
73
|
+
overhead=len(canonical(packet))+48
|
|
74
|
+
path_prefix=len('/-/g/transfer.part_put/j/')
|
|
75
|
+
path_budget=(path_bytes-path_prefix)*3//4-overhead
|
|
76
|
+
request_budget=request_bytes-overhead
|
|
77
|
+
response_budget=response_bytes-1024
|
|
78
|
+
chosen=min(app.settings.max_part_bytes,a.get('requested_part_bytes',app.settings.max_part_bytes),
|
|
79
|
+
request_budget*3//4,path_budget*3//4,response_budget*3//4)
|
|
80
|
+
require(chosen>=1,'transport_limit_too_small')
|
|
81
|
+
return {'part_bytes':chosen,'max_request_bytes':request_bytes,'max_response_bytes':response_bytes,
|
|
82
|
+
'max_path_bytes':path_bytes}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def install(app):
|
|
86
|
+
op,finish=registration(app,'transfer',('identity','content'))
|
|
87
|
+
|
|
88
|
+
def service():
|
|
89
|
+
async def publish(ctx,tx,parent,blob):
|
|
90
|
+
# An internal operation label is not a new or user-supplied principal.
|
|
91
|
+
operation=SimpleNamespace(operation='transfer.seal',contract_version=1)
|
|
92
|
+
resource=await create_resource(app,ctx,operation,tx,parent=parent,type='file',body=blob,
|
|
93
|
+
media_type=blob.media_type,mode=0o600)
|
|
94
|
+
return ResourceRef(id=resource.id,revision=resource.revision)
|
|
95
|
+
return TransferService(app.metadata,app.contents,app.authorizer,app.clock,publish=publish,
|
|
96
|
+
ttl=app.settings.transfer_ttl,part_bytes=app.settings.max_part_bytes)
|
|
97
|
+
|
|
98
|
+
@op('transfer.open',obj({'direction':{'enum':['upload','download']},'target':REF,'size':SIZE,'digest':DIGEST,
|
|
99
|
+
'media_type':STRING,'requested_part_bytes':{'type':'integer','minimum':1},
|
|
100
|
+
'max_request_bytes':{'type':'integer','minimum':1},'max_response_bytes':{'type':'integer','minimum':1},
|
|
101
|
+
'max_path_bytes':{'type':'integer','minimum':1}},('direction',)))
|
|
102
|
+
async def open_transfer(ctx,request,tx):
|
|
103
|
+
a=request.arguments
|
|
104
|
+
target=decode(ResourceRef,a['target']) if 'target' in a else None
|
|
105
|
+
if target is not None and target.id.startswith('/'):
|
|
106
|
+
target=replace(target,id=await resolve(tx,target.id))
|
|
107
|
+
limits=negotiate(app,request)
|
|
108
|
+
transfer=await service().open(ctx,a['direction'],target,a.get('size'),a.get('digest'),
|
|
109
|
+
ctx.now+timedelta(seconds=app.settings.transfer_ttl),limits=limits,
|
|
110
|
+
media_type=a.get('media_type','application/octet-stream'))
|
|
111
|
+
return HandlerOutput(data={'transfer_id':transfer.id,'direction':transfer.direction,'state':transfer.state,
|
|
112
|
+
'target':wire(transfer.target),'size':transfer.expected_size,'digest':transfer.expected_digest,
|
|
113
|
+
'expires_at':wire(transfer.expires_at),**limits})
|
|
114
|
+
|
|
115
|
+
@op('transfer.part_put',obj({'transfer_id':IDENTIFIER,'offset':SIZE,'data':BYTES,'digest':DIGEST},
|
|
116
|
+
('transfer_id','offset','data','digest')))
|
|
117
|
+
async def put(ctx,request,tx):
|
|
118
|
+
a=request.arguments
|
|
119
|
+
chunk=await service().put(ctx,a['transfer_id'],a['offset'],unb64(a['data'],limit=app.settings.max_part_bytes),a['digest'])
|
|
120
|
+
return HandlerOutput(data={'chunk':wire(chunk)})
|
|
121
|
+
|
|
122
|
+
@op('transfer.part_get',obj({'transfer_id':IDENTIFIER,'offset':SIZE,'length':SIZE},
|
|
123
|
+
('transfer_id','offset','length')),effect='read')
|
|
124
|
+
async def get(ctx,request,tx):
|
|
125
|
+
a=request.arguments
|
|
126
|
+
chunk,data=await service().get(ctx,a['transfer_id'],(a['offset'],a['offset']+a['length']))
|
|
127
|
+
return HandlerOutput(data={'chunk':wire(chunk),'data':b64(data)})
|
|
128
|
+
|
|
129
|
+
@op('transfer.status',obj({'transfer_id':IDENTIFIER,'cursor':STRING,'limit':{'type':'integer','minimum':1,'maximum':500}},
|
|
130
|
+
('transfer_id',)),effect='read')
|
|
131
|
+
async def status(ctx,request,tx):
|
|
132
|
+
a=request.arguments
|
|
133
|
+
page=await service().status(ctx,a['transfer_id'],a.get('cursor'),a.get('limit',50))
|
|
134
|
+
transfer=await tx.transfer(a['transfer_id'])
|
|
135
|
+
output={'transfer_id':transfer.id,'state':transfer.state,'direction':transfer.direction,
|
|
136
|
+
'generation':transfer.generation,'size':transfer.expected_size,'digest':transfer.expected_digest,
|
|
137
|
+
'missing':wire(page.items),'next_cursor':page.next_cursor,'output':wire(transfer.output)}
|
|
138
|
+
if transfer.expected_size is None:
|
|
139
|
+
after=int(a.get('cursor','0'))
|
|
140
|
+
rows=tx.rows('SELECT offset,length FROM chunks WHERE transfer_id=? AND offset>=? ORDER BY offset LIMIT ?',
|
|
141
|
+
(transfer.id,after,a.get('limit',50)+1))
|
|
142
|
+
selected=rows[:a.get('limit',50)]
|
|
143
|
+
output['received']=[[row[0],row[0]+row[1]] for row in selected]
|
|
144
|
+
output['next_cursor']=str(rows[-1][0]) if len(rows)>len(selected) else None
|
|
145
|
+
return HandlerOutput(data=output)
|
|
146
|
+
|
|
147
|
+
@op('transfer.seal',obj({'transfer_id':IDENTIFIER,'final_size':SIZE,'final_digest':DIGEST},
|
|
148
|
+
('transfer_id','final_size','final_digest')))
|
|
149
|
+
async def seal(ctx,request,tx):
|
|
150
|
+
a=request.arguments
|
|
151
|
+
ref=await service().seal(ctx,a['transfer_id'],a['final_size'],a['final_digest'])
|
|
152
|
+
return HandlerOutput(resources=(ref,),data={'transfer_id':a['transfer_id'],'state':'sealed',
|
|
153
|
+
'size':a['final_size'],'digest':a['final_digest']},output=ref)
|
|
154
|
+
|
|
155
|
+
@op('transfer.cancel',obj({'transfer_id':IDENTIFIER},('transfer_id',)))
|
|
156
|
+
async def cancel(ctx,request,tx):
|
|
157
|
+
await service().cancel(ctx,request.arguments['transfer_id'])
|
|
158
|
+
return HandlerOutput(data={'transfer_id':request.arguments['transfer_id'],'state':'cancelled'})
|
|
159
|
+
|
|
160
|
+
@op('transfer.query_seal',obj({'transfer_id':IDENTIFIER},('transfer_id',)))
|
|
161
|
+
async def query_seal(ctx,request,tx):
|
|
162
|
+
transfer=await tx.transfer(request.arguments['transfer_id'])
|
|
163
|
+
ref,revision,kind,_=await sealed_read_query(app,ctx,request,tx,transfer)
|
|
164
|
+
expiry=min(transfer.expires_at,ctx.now+timedelta(minutes=15))
|
|
165
|
+
source=await tx.resource(ref.id)
|
|
166
|
+
marker_key='query_ref_source:'+ref.id
|
|
167
|
+
previous=tx.setting(marker_key)
|
|
168
|
+
retention=max(transfer.expires_at,expiry)+timedelta(hours=1)
|
|
169
|
+
if previous is not None:
|
|
170
|
+
retention=max(retention,parse_time(previous['retain_until']))
|
|
171
|
+
tx.set_setting(marker_key,{'transfer_id':transfer.id,'subject':ctx.principal.subject,
|
|
172
|
+
'revision':ref.revision,'digest':revision.content.digest,'parent':source.parent,
|
|
173
|
+
'retain_until':wire(retention)})
|
|
174
|
+
token=app.cursors.encode('query-ref',
|
|
175
|
+
{'service':app.settings.service_url,'transfer_id':transfer.id,
|
|
176
|
+
'output':wire(ref),'digest':revision.content.digest,'query_kind':kind},
|
|
177
|
+
{'principal':query_ref_principal(ctx.principal),'expires_at':wire(expiry)})
|
|
178
|
+
return HandlerOutput(data={'query_ref':token,'expires_at':wire(expiry),
|
|
179
|
+
'next':'/_r/q/'+token})
|
|
180
|
+
|
|
181
|
+
@op('transfer.query_get',obj({'query_ref':STRING,'cursor':STRING},('query_ref',)),effect='read')
|
|
182
|
+
async def query_get(ctx,request,tx):
|
|
183
|
+
token=request.arguments['query_ref']
|
|
184
|
+
signed=app.cursors.inspect(token)
|
|
185
|
+
require(signed.get('kind')=='query-ref','invalid_query_ref')
|
|
186
|
+
query,position=signed['query'],signed['position']
|
|
187
|
+
require(query.get('service')==app.settings.service_url,'wrong_service')
|
|
188
|
+
require(position.get('principal')==query_ref_principal(ctx.principal),
|
|
189
|
+
'query_ref_principal_mismatch')
|
|
190
|
+
require(ctx.now<parse_time(position['expires_at']),'query_ref_expired')
|
|
191
|
+
transfer=await tx.transfer(query['transfer_id'])
|
|
192
|
+
ref,revision,kind,args=await sealed_read_query(app,ctx,request,tx,transfer)
|
|
193
|
+
require(query.get('output')==wire(ref) and query.get('digest')==revision.content.digest,
|
|
194
|
+
'query_ref_digest_mismatch')
|
|
195
|
+
require(query.get('query_kind')==kind,'query_ref_digest_mismatch')
|
|
196
|
+
operation='discovery.read_query' if kind=='read' else 'discovery.lexical_search'
|
|
197
|
+
contract_version=(read_query_version(args) if kind=='read' else
|
|
198
|
+
4 if kind=='search' and 'suggest' in args else
|
|
199
|
+
3 if kind=='search' and any(name in args for name in
|
|
200
|
+
('source_kind','relation_type')) else 2 if kind=='search' and 'facets' in args else 1)
|
|
201
|
+
principal=query_ref_principal(ctx.principal)
|
|
202
|
+
normalized=dict(args)
|
|
203
|
+
if kind=='read' and contract_version==3:
|
|
204
|
+
normalized['query_version']=3
|
|
205
|
+
scope_field='parent' if kind=='read' else 'scope'
|
|
206
|
+
if normalized.get(scope_field):
|
|
207
|
+
normalized[scope_field]=await resolve(tx,normalized[scope_field])
|
|
208
|
+
if normalized.get('tag'):
|
|
209
|
+
normalized['tag']=normalize_tag(normalized['tag'])
|
|
210
|
+
nested_args=args
|
|
211
|
+
internal_page=None
|
|
212
|
+
if request.arguments.get('cursor'):
|
|
213
|
+
compact=app.cursors.inspect(request.arguments['cursor'])
|
|
214
|
+
require(compact.get('kind')=='query-ref-page' and
|
|
215
|
+
compact.get('query',{}).get('query_ref_digest')==digest(token),
|
|
216
|
+
'invalid_cursor')
|
|
217
|
+
page=compact['position']
|
|
218
|
+
require(page.get('principal')==principal,'query_ref_principal_mismatch')
|
|
219
|
+
require(ctx.now<parse_time(page['expires_at']),'cursor_expired')
|
|
220
|
+
require(page.get('query_digest')==digest(normalized),'cursor_query_mismatch')
|
|
221
|
+
internal_page={'arguments':normalized,'last':page['last'],
|
|
222
|
+
'snapshot':page['snapshot']}
|
|
223
|
+
# Reuse the installed read handler and its current per-object Authorizer.
|
|
224
|
+
# Query bytes can select only this registered read operation.
|
|
225
|
+
nested=SimpleNamespace(operation=operation,contract_version=contract_version,
|
|
226
|
+
arguments=nested_args,internal_page_state=internal_page)
|
|
227
|
+
output=await app.registry.operation(operation,contract_version).handler(ctx,nested,tx)
|
|
228
|
+
data=dict(output.data)
|
|
229
|
+
if data.get('cursor'):
|
|
230
|
+
# This oversized cursor was generated in this call by the trusted
|
|
231
|
+
# read handler. Never expose it or raise the public 8 KiB limit.
|
|
232
|
+
encoded=data['cursor'].split('.',1)[0]
|
|
233
|
+
generated=loads(unb64(encoded,limit=131072))
|
|
234
|
+
require(generated.get('kind')=='read-page','invalid_cursor')
|
|
235
|
+
page_query,page=generated['query'],generated['position']
|
|
236
|
+
require(page_query=={'operation':operation,'arguments':normalized},
|
|
237
|
+
'cursor_query_mismatch')
|
|
238
|
+
expiry=min(parse_time(position['expires_at']),parse_time(page['expires_at']))
|
|
239
|
+
short=app.cursors.encode('query-ref-page',{'query_ref_digest':digest(token)},
|
|
240
|
+
{'last':page['last'],'snapshot':page['snapshot'],'principal':principal,
|
|
241
|
+
'query_digest':digest(normalized),'expires_at':wire(expiry)})
|
|
242
|
+
data['cursor']=short
|
|
243
|
+
if data.get('next'):
|
|
244
|
+
data.update(next=f'/_r/q/{token}/c/{short}',next_requires_auth=True)
|
|
245
|
+
if 'pageInfo' in data:
|
|
246
|
+
data['pageInfo']={**data['pageInfo'],'endCursor':short}
|
|
247
|
+
return replace(output,data=data)
|
|
248
|
+
|
|
249
|
+
finish()
|
msg/py.typed
ADDED
|
File without changes
|
msg/security/__init__.py
ADDED
|
File without changes
|
msg/security/age_keys.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""age X25519 key text encoding. This module does not encrypt age payloads."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
|
|
6
|
+
from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey
|
|
7
|
+
|
|
8
|
+
from msg.core.errors import Failure, require
|
|
9
|
+
|
|
10
|
+
ALPHABET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l'
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _polymod(values):
|
|
14
|
+
generators=(0x3b6a57b2,0x26508e6d,0x1ea119fa,0x3d4233dd,0x2a1462b3)
|
|
15
|
+
check=1
|
|
16
|
+
for value in values:
|
|
17
|
+
top=check>>25
|
|
18
|
+
check=((check&0x1ffffff)<<5)^value
|
|
19
|
+
for index,generator in enumerate(generators):
|
|
20
|
+
if (top>>index)&1:
|
|
21
|
+
check^=generator
|
|
22
|
+
return check
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _hrp_expand(hrp):
|
|
26
|
+
return [ord(char)>>5 for char in hrp]+[0]+[ord(char)&31 for char in hrp]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _convert(data,from_bits,to_bits, *, pad):
|
|
30
|
+
accumulator=bits=0
|
|
31
|
+
result=[]
|
|
32
|
+
maximum=(1<<to_bits)-1
|
|
33
|
+
for value in data:
|
|
34
|
+
require(0<=value<(1<<from_bits),'invalid_age_key')
|
|
35
|
+
accumulator=(accumulator<<from_bits)|value
|
|
36
|
+
bits+=from_bits
|
|
37
|
+
while bits>=to_bits:
|
|
38
|
+
bits-=to_bits
|
|
39
|
+
result.append((accumulator>>bits)&maximum)
|
|
40
|
+
if pad and bits:
|
|
41
|
+
result.append((accumulator<<(to_bits-bits))&maximum)
|
|
42
|
+
elif not pad:
|
|
43
|
+
require(bits<from_bits and (accumulator<<(to_bits-bits))&maximum==0,'invalid_age_key')
|
|
44
|
+
return bytes(result)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _encode(hrp,payload):
|
|
48
|
+
values=_convert(payload,8,5,pad=True)
|
|
49
|
+
checksum=_polymod([*_hrp_expand(hrp),*values,0,0,0,0,0,0])^1
|
|
50
|
+
check=[(checksum>>(5*(5-index)))&31 for index in range(6)]
|
|
51
|
+
return hrp+'1'+''.join(ALPHABET[value] for value in (*values,*check))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _decode(value,hrp):
|
|
55
|
+
require(type(value) is str and value.lower()==value and value.startswith(hrp+'1'),
|
|
56
|
+
'invalid_age_recipient')
|
|
57
|
+
characters=value[len(hrp)+1:]
|
|
58
|
+
require(len(characters)==58 and all(char in ALPHABET for char in characters),
|
|
59
|
+
'invalid_age_recipient')
|
|
60
|
+
numbers=[ALPHABET.index(char) for char in characters]
|
|
61
|
+
require(_polymod([*_hrp_expand(hrp),*numbers])==1,'invalid_age_recipient')
|
|
62
|
+
raw=_convert(numbers[:-6],5,8,pad=False)
|
|
63
|
+
require(len(raw)==32,'invalid_age_recipient')
|
|
64
|
+
return raw
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def recipient_from_public(public: bytes):
|
|
68
|
+
require(type(public) is bytes and len(public)==32,'invalid_encryption_public_key')
|
|
69
|
+
return _encode('age',public)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def public_from_recipient(recipient: str):
|
|
73
|
+
public=_decode(recipient,'age')
|
|
74
|
+
try:
|
|
75
|
+
# Reject low-order points which cannot produce an X25519 shared secret.
|
|
76
|
+
X25519PrivateKey.generate().exchange(X25519PublicKey.from_public_bytes(public))
|
|
77
|
+
except ValueError as exc:
|
|
78
|
+
raise Failure('invalid_age_recipient') from exc
|
|
79
|
+
return public
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def encryption_key_id(public: bytes):
|
|
83
|
+
require(type(public) is bytes and len(public)==32,'invalid_encryption_public_key')
|
|
84
|
+
return 'ek_'+hashlib.sha256(b'msg-encryption-subkey-v1\0'+public).hexdigest()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def generate_age_key():
|
|
88
|
+
private=X25519PrivateKey.generate()
|
|
89
|
+
secret=private.private_bytes_raw()
|
|
90
|
+
public=private.public_key().public_bytes_raw()
|
|
91
|
+
# age's X25519 identity is upper-case Bech32; the recipient is lower-case.
|
|
92
|
+
identity=_encode('age-secret-key-',secret).upper()
|
|
93
|
+
return identity,recipient_from_public(public)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def recipient_from_identity(identity: str):
|
|
97
|
+
require(type(identity) is str and identity==identity.upper(),'invalid_age_identity')
|
|
98
|
+
try:
|
|
99
|
+
private=_decode(identity.lower(),'age-secret-key-')
|
|
100
|
+
public=X25519PrivateKey.from_private_bytes(private).public_key().public_bytes_raw()
|
|
101
|
+
except Failure as exc:
|
|
102
|
+
raise Failure('invalid_age_identity') from exc
|
|
103
|
+
return recipient_from_public(public)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def private_from_identity(identity: str):
|
|
107
|
+
require(type(identity) is str and identity==identity.upper(),'invalid_age_identity')
|
|
108
|
+
try:
|
|
109
|
+
private=X25519PrivateKey.from_private_bytes(_decode(identity.lower(),'age-secret-key-'))
|
|
110
|
+
except (Failure,ValueError) as exc:
|
|
111
|
+
raise Failure('invalid_age_identity') from exc
|
|
112
|
+
return private
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Principal construction is a trusted server operation, never request data."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import hmac
|
|
6
|
+
from dataclasses import replace
|
|
7
|
+
from msg.constants import ROOT_SUBJECT
|
|
8
|
+
from msg.core.codec import digest,unb64,parse_time
|
|
9
|
+
from msg.core.errors import Failure,require
|
|
10
|
+
from msg.core.models import Principal,SignatureProof,TokenProof
|
|
11
|
+
from msg.core.requests import payload_fields,signing_bytes
|
|
12
|
+
from msg.security.crypto import key_id,subject_id,verify
|
|
13
|
+
from msg.security.quarantine import active as quarantine_active
|
|
14
|
+
from msg.security.token_delivery import recovery_verifier
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
CUSTODIAL_SIGNED_WRITES=frozenset({
|
|
18
|
+
'content.post_create','content.post_edit','content.file_put','content.attach',
|
|
19
|
+
'discussion.reply','discussion.quote','discussion.repost','identity.token_rotate',
|
|
20
|
+
'identity.custodial_upgrade_start','identity.custodial_upgrade_finish',
|
|
21
|
+
'identity.custodial_rewrap_entry','identity.custodial_rewrap_revision',
|
|
22
|
+
'identity.custodial_rewrap_ack',
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class AuthenticationService:
|
|
27
|
+
def __init__(self,registry,certificates,service,clock,primary_ceiling,temporary_ceiling):
|
|
28
|
+
self.registry,self.certificates,self.service,self.clock=registry,certificates,service,clock
|
|
29
|
+
self.primary_ceiling,self.temporary_ceiling=primary_ceiling,temporary_ceiling
|
|
30
|
+
|
|
31
|
+
async def authenticate(self,request,session, *, entry):
|
|
32
|
+
require(session.setting('active_root_certificate',self.certificates.root_certificate.resource_id)==
|
|
33
|
+
self.certificates.root_certificate.resource_id,'service_restart_required')
|
|
34
|
+
spec=self.registry.operation(request.operation,request.contract_version)
|
|
35
|
+
if quarantine_active(session):
|
|
36
|
+
require(spec.effect=='read','writes_paused')
|
|
37
|
+
raise Failure('recovery_quarantined')
|
|
38
|
+
require(request.protocol_version==1 and request.target_service==self.service,'wrong_service')
|
|
39
|
+
require(request.payload_digest==digest(payload_fields(request)),'payload_digest_mismatch')
|
|
40
|
+
now=self.clock()
|
|
41
|
+
if spec.effect!='read' or request.proof is not None:
|
|
42
|
+
require(request.expires_at is not None and now<request.expires_at,'request_expired')
|
|
43
|
+
require((request.expires_at-now).total_seconds()<=300,'request_expiry_too_long')
|
|
44
|
+
proof=request.proof
|
|
45
|
+
if request.operation=='identity.register':
|
|
46
|
+
require(isinstance(proof,SignatureProof),'proof_required')
|
|
47
|
+
public=unb64(request.arguments['public_key'],limit=32)
|
|
48
|
+
require(len(public)==32 and request.subject==subject_id(public),'subject_mismatch')
|
|
49
|
+
verify(public,signing_bytes(request),proof.signature,purpose='request')
|
|
50
|
+
return Principal(actor=request.subject,subject=request.subject,credential_id=key_id(public),
|
|
51
|
+
method='signature',certificates=(),ceiling=self.primary_ceiling())
|
|
52
|
+
if request.operation in {'identity.temporary','identity.custodial_create'}:
|
|
53
|
+
require(proof is None and request.subject is None,'invalid_bootstrap')
|
|
54
|
+
nonce=unb64(request.arguments['nonce'],limit=64)
|
|
55
|
+
require(len(nonce)>=24,'invalid_bootstrap_nonce')
|
|
56
|
+
# The high-entropy claim defines the initial temporary subject; ordinary anonymous
|
|
57
|
+
# reads cannot create it. The token itself is derived only by the server.
|
|
58
|
+
prefix='u_cust_' if request.operation=='identity.custodial_create' else 'u_tmp_'
|
|
59
|
+
subject=prefix+hashlib.sha256(nonce).hexdigest()[:32]
|
|
60
|
+
return Principal(actor=subject,subject=subject,credential_id='t_'+subject[2:],method='token',
|
|
61
|
+
certificates=(),ceiling=self.temporary_ceiling())
|
|
62
|
+
if request.operation=='identity.token_recover':
|
|
63
|
+
require(proof is None and request.subject is not None,'invalid_recovery_proof')
|
|
64
|
+
old=await session.credential(request.arguments['credential_id'])
|
|
65
|
+
require(old.kind=='token' and old.subject_id==request.subject,'recovery_unavailable')
|
|
66
|
+
actor=await session.subject(old.subject_id)
|
|
67
|
+
require(not actor.local_only and actor.resource_id!=ROOT_SUBJECT,'local_only')
|
|
68
|
+
verifier=recovery_verifier(request.arguments['recovery_secret'])
|
|
69
|
+
row=session.one('''SELECT recovery_verifier,recovery_expires_at,consumed_at,request_id
|
|
70
|
+
FROM token_deliveries WHERE credential_id=? AND subject=?''',
|
|
71
|
+
(old.id,actor.resource_id))
|
|
72
|
+
require(row is not None and hmac.compare_digest(row[0],verifier) and
|
|
73
|
+
parse_time(row[1])>now and row[3]==request.arguments['original_request_id'],
|
|
74
|
+
'recovery_unavailable')
|
|
75
|
+
if row[2] is not None:
|
|
76
|
+
previous=await session.request_result(actor.resource_id,request.request_id,request.payload_digest)
|
|
77
|
+
require(previous is not None and previous.data.get('previous_credential')==old.id,
|
|
78
|
+
'recovery_unavailable')
|
|
79
|
+
else:
|
|
80
|
+
require(old.revoked_at is None and old.expires_at>now,'recovery_unavailable')
|
|
81
|
+
operation='identity.token_recover@1'
|
|
82
|
+
ceiling=tuple(replace(g,operations=frozenset({operation})) for g in self.primary_ceiling()
|
|
83
|
+
if g.capability=='identity.basic' and operation in g.operations)
|
|
84
|
+
require(bool(ceiling),'recovery_unavailable')
|
|
85
|
+
return Principal(actor=actor.resource_id,subject=actor.resource_id,
|
|
86
|
+
credential_id=old.id,method='recovery',certificates=(),ceiling=ceiling)
|
|
87
|
+
if proof is None:
|
|
88
|
+
require(request.subject is None and spec.effect=='read','authentication_required')
|
|
89
|
+
return Principal(actor=None,subject=None,credential_id=None,method='anonymous',certificates=(),ceiling=())
|
|
90
|
+
credential_id=proof.signature.key_id if isinstance(proof,SignatureProof) else proof.credential_id
|
|
91
|
+
credential=await session.credential(credential_id)
|
|
92
|
+
actor=await session.subject(credential.subject_id)
|
|
93
|
+
require(credential.not_before<=now and (credential.expires_at is None or now<credential.expires_at),
|
|
94
|
+
'credential_expired')
|
|
95
|
+
if isinstance(proof,SignatureProof):
|
|
96
|
+
require(credential.kind=='signing_key','wrong_credential_kind')
|
|
97
|
+
verify(credential.verifier,signing_bytes(request),proof.signature,purpose='request')
|
|
98
|
+
method='signature'
|
|
99
|
+
ids=proof.certificates
|
|
100
|
+
elif isinstance(proof,TokenProof):
|
|
101
|
+
require(credential.kind=='token','wrong_credential_kind')
|
|
102
|
+
require(hmac.compare_digest(credential.verifier,hashlib.sha256(proof.token).digest()),'invalid_token')
|
|
103
|
+
method='token'
|
|
104
|
+
ids=()
|
|
105
|
+
else:
|
|
106
|
+
raise Failure('invalid_proof')
|
|
107
|
+
if credential.revoked_at is not None:
|
|
108
|
+
# A token invalidated by its own successful rotation may only retrieve
|
|
109
|
+
# that exact committed rotation result. It cannot authorize new work.
|
|
110
|
+
previous=None
|
|
111
|
+
if method=='token' and request.operation in {
|
|
112
|
+
'identity.token_rotate','identity.custodial_upgrade_finish'}:
|
|
113
|
+
previous=await session.request_result(actor.resource_id,request.request_id,request.payload_digest)
|
|
114
|
+
require(previous is not None and previous.data.get('previous_credential')==credential.id,
|
|
115
|
+
'credential_revoked')
|
|
116
|
+
successor=await session.credential(previous.data['credential_id'])
|
|
117
|
+
require(successor.subject_id==actor.resource_id and successor.revoked_at is None and
|
|
118
|
+
(successor.expires_at is None or successor.expires_at>now),'credential_revoked')
|
|
119
|
+
subject=await session.subject(request.subject or actor.resource_id)
|
|
120
|
+
require(not (actor.local_only or subject.local_only or actor.resource_id==ROOT_SUBJECT or
|
|
121
|
+
subject.resource_id==ROOT_SUBJECT) or entry=='local_admin','local_only')
|
|
122
|
+
require(entry in spec.entries,'entry_not_allowed')
|
|
123
|
+
operation=f'{spec.name}@{spec.version}'
|
|
124
|
+
require(any(operation in g.operations for g in credential.ceiling),'credential_ceiling')
|
|
125
|
+
if actor.kind=='custodial' and method=='token' and spec.effect!='read':
|
|
126
|
+
download_only=(spec.name=='transfer.open' and
|
|
127
|
+
request.arguments.get('direction')=='download')
|
|
128
|
+
require(spec.name in CUSTODIAL_SIGNED_WRITES or download_only,
|
|
129
|
+
'custodial_operation_not_supported')
|
|
130
|
+
require(not spec.require_signature or method=='signature','signature_required')
|
|
131
|
+
valid=[]
|
|
132
|
+
for cid in ids:
|
|
133
|
+
cert=await self.certificates.validate(cid,session)
|
|
134
|
+
require(cert.subject_id==actor.resource_id and cert.key_id==credential.id,'certificate_subject_mismatch')
|
|
135
|
+
valid.append(cert)
|
|
136
|
+
if actor.resource_id!=subject.resource_id:
|
|
137
|
+
require(any(c.kind=='delegation' and any(
|
|
138
|
+
(session.setting('delegation:'+s.id) or {}).get('grantor')==subject.resource_id
|
|
139
|
+
for s in c.authority_sources) for c in valid),'delegation_required')
|
|
140
|
+
return Principal(actor=actor.resource_id,subject=subject.resource_id,credential_id=credential.id,
|
|
141
|
+
method=method,certificates=tuple(c.resource_id for c in valid),ceiling=credential.ceiling)
|