msgctl 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. msg/__init__.py +2 -0
  2. msg/admin/__init__.py +0 -0
  3. msg/admin/backup_retirement.py +56 -0
  4. msg/admin/backups.py +341 -0
  5. msg/admin/custodial_check.py +77 -0
  6. msg/admin/diagnostics.py +750 -0
  7. msg/admin/market.py +91 -0
  8. msg/admin/market_check.py +341 -0
  9. msg/admin/money.py +322 -0
  10. msg/admin/preflight.py +142 -0
  11. msg/admin/recovery_replay.py +281 -0
  12. msg/admin/restore_database.py +37 -0
  13. msg/admin/root.py +347 -0
  14. msg/admin/rotation.py +197 -0
  15. msg/admin/token_delivery_check.py +54 -0
  16. msg/admin/upgrade_check.py +58 -0
  17. msg/application.py +163 -0
  18. msg/bootstrap.py +305 -0
  19. msg/cli.py +537 -0
  20. msg/client.py +725 -0
  21. msg/client_certificates.py +98 -0
  22. msg/client_content.py +43 -0
  23. msg/client_custodial.py +206 -0
  24. msg/client_market.py +104 -0
  25. msg/client_recovery.py +186 -0
  26. msg/client_secrets.py +58 -0
  27. msg/client_tokens.py +99 -0
  28. msg/client_upgrade.py +188 -0
  29. msg/config.py +318 -0
  30. msg/constants.py +11 -0
  31. msg/core/__init__.py +0 -0
  32. msg/core/batching.py +23 -0
  33. msg/core/codec.py +253 -0
  34. msg/core/contracts.py +128 -0
  35. msg/core/cursors.py +68 -0
  36. msg/core/email_address.py +33 -0
  37. msg/core/errors.py +22 -0
  38. msg/core/events.py +6 -0
  39. msg/core/execution_ports.py +20 -0
  40. msg/core/executor.py +193 -0
  41. msg/core/models.py +517 -0
  42. msg/core/packet.py +71 -0
  43. msg/core/permissions.py +14 -0
  44. msg/core/query.py +44 -0
  45. msg/core/read_query.py +60 -0
  46. msg/core/registry.py +173 -0
  47. msg/core/requests.py +58 -0
  48. msg/core/schema_policy.py +28 -0
  49. msg/core/schemas.py +30 -0
  50. msg/core/tags.py +21 -0
  51. msg/core/template_dsl.py +94 -0
  52. msg/core/text_patch.py +283 -0
  53. msg/core/tool_execution.py +28 -0
  54. msg/core/transfer.py +183 -0
  55. msg/daemon.py +255 -0
  56. msg/data/__init__.py +0 -0
  57. msg/data/bootstrap.json +72 -0
  58. msg/data/favicon.png +0 -0
  59. msg/data/logo-dark.svg +5 -0
  60. msg/data/logo.svg +5 -0
  61. msg/data/recovery-checkpoint.example.json +1 -0
  62. msg/data/recovery-checkpoint.schema.json +118 -0
  63. msg/data/shortcodes.json +1 -0
  64. msg/data/system/AGENTS.md +6 -0
  65. msg/data/system/rules/_index.md +17 -0
  66. msg/data/system/rules/auth.md +6 -0
  67. msg/data/system/rules/files.md +6 -0
  68. msg/data/system/rules/identity.md +6 -0
  69. msg/data/system/rules/protocol.md +6 -0
  70. msg/data/system/rules/read-write.md +6 -0
  71. msg/data/system/rules/recovery.md +6 -0
  72. msg/data/system/rules/security.md +10 -0
  73. msg/data/system/rules/topics.md +6 -0
  74. msg/extensions/__init__.py +1 -0
  75. msg/extensions/hosting.py +259 -0
  76. msg/extensions/keystore.py +96 -0
  77. msg/extensions/repositories.py +769 -0
  78. msg/extensions/rss.py +67 -0
  79. msg/extensions/ssh.py +264 -0
  80. msg/extensions/ssh_git.py +300 -0
  81. msg/extensions/tools.py +150 -0
  82. msg/hosting_runtime.py +138 -0
  83. msg/market/__init__.py +1 -0
  84. msg/market/arbitration.py +481 -0
  85. msg/market/delivery.py +338 -0
  86. msg/market/delivery_notifications.py +110 -0
  87. msg/market/delivery_targets.py +76 -0
  88. msg/market/email.py +100 -0
  89. msg/market/escrow.py +345 -0
  90. msg/market/orders.py +218 -0
  91. msg/market/policy.py +147 -0
  92. msg/market/rationale.py +91 -0
  93. msg/market/references.py +19 -0
  94. msg/market/targets.py +157 -0
  95. msg/plugins/__init__.py +28 -0
  96. msg/plugins/achievements.py +316 -0
  97. msg/plugins/batch.py +32 -0
  98. msg/plugins/bounty.py +366 -0
  99. msg/plugins/collaboration.py +261 -0
  100. msg/plugins/common.py +217 -0
  101. msg/plugins/communication.py +853 -0
  102. msg/plugins/content.py +855 -0
  103. msg/plugins/custodial_lifecycle.py +224 -0
  104. msg/plugins/delivery.py +230 -0
  105. msg/plugins/discovery.py +1266 -0
  106. msg/plugins/discussion.py +162 -0
  107. msg/plugins/extensions.py +10 -0
  108. msg/plugins/following.py +78 -0
  109. msg/plugins/hosting_capacity.py +52 -0
  110. msg/plugins/identity.py +1654 -0
  111. msg/plugins/money.py +221 -0
  112. msg/plugins/offers.py +183 -0
  113. msg/plugins/orders.py +273 -0
  114. msg/plugins/recovery.py +376 -0
  115. msg/plugins/schemas.py +5 -0
  116. msg/plugins/sharing.py +300 -0
  117. msg/plugins/store.py +249 -0
  118. msg/plugins/system.py +75 -0
  119. msg/plugins/transfer.py +249 -0
  120. msg/py.typed +0 -0
  121. msg/security/__init__.py +0 -0
  122. msg/security/age_keys.py +112 -0
  123. msg/security/authentication.py +141 -0
  124. msg/security/authorization.py +300 -0
  125. msg/security/backup_retirement.py +135 -0
  126. msg/security/capabilities.py +157 -0
  127. msg/security/certificates.py +183 -0
  128. msg/security/crypto.py +102 -0
  129. msg/security/custodial_migration.py +292 -0
  130. msg/security/custody_history.py +146 -0
  131. msg/security/network.py +81 -0
  132. msg/security/policy.py +67 -0
  133. msg/security/quarantine.py +19 -0
  134. msg/security/root_files.py +67 -0
  135. msg/security/rotation_journal.py +60 -0
  136. msg/security/sealed_box.py +70 -0
  137. msg/security/sharing_policy.py +29 -0
  138. msg/security/token_delivery.py +93 -0
  139. msg/security/vault.py +186 -0
  140. msg/storage/__init__.py +0 -0
  141. msg/storage/capacity.py +100 -0
  142. msg/storage/custodial_migration.py +22 -0
  143. msg/storage/git.py +435 -0
  144. msg/storage/ledger_migration.py +307 -0
  145. msg/storage/market_migration.py +49 -0
  146. msg/storage/postgres.py +663 -0
  147. msg/storage/query.py +39 -0
  148. msg/storage/read_only.py +126 -0
  149. msg/storage/session.py +403 -0
  150. msg/storage/sqlite.py +360 -0
  151. msg/storage/topic_event_migration.py +26 -0
  152. msg/storage/valkey_bus.py +45 -0
  153. msg/transports/__init__.py +0 -0
  154. msg/transports/client.py +183 -0
  155. msg/transports/dictionary.py +387 -0
  156. msg/transports/graphql.py +75 -0
  157. msg/transports/http.py +72 -0
  158. msg/transports/http_routes.py +1577 -0
  159. msg/transports/mcp.py +127 -0
  160. msg/transports/packet.py +42 -0
  161. msg/transports/read_tree_path.py +65 -0
  162. msg/transports/stdio.py +36 -0
  163. msg/transports/url_safety.py +123 -0
  164. msg/tui.py +368 -0
  165. msg/workers/__init__.py +1 -0
  166. msg/workers/effects.py +406 -0
  167. msg/workers/leases.py +19 -0
  168. msg/workers/mail.py +63 -0
  169. msg/workers/maintenance.py +310 -0
  170. msg/workers/sandbox.py +74 -0
  171. msg/workers/sandbox_child.py +189 -0
  172. msg/workers/webhook.py +160 -0
  173. msgctl-0.1.0a1.dist-info/METADATA +96 -0
  174. msgctl-0.1.0a1.dist-info/RECORD +176 -0
  175. msgctl-0.1.0a1.dist-info/WHEEL +4 -0
  176. msgctl-0.1.0a1.dist-info/entry_points.txt +4 -0
msg/extensions/rss.py ADDED
@@ -0,0 +1,67 @@
1
+ """Read-only RSS 2.0 projection over the same resource visibility checks."""
2
+ from __future__ import annotations
3
+ from email.utils import format_datetime
4
+ from xml.etree import ElementTree as ET
5
+ from msg.core.codec import canonical, digest, wire, decode, loads
6
+ from msg.core.models import Resource, ResourceRef, HandlerOutput
7
+ from msg.plugins.common import resolve, check_access
8
+ from msg.plugins.discovery import visible, next_link
9
+ from msg.plugins.schemas import obj, IDENTIFIER, STRING
10
+
11
+
12
+ def register(app, op):
13
+ @op('discovery.feed', obj({'parent':IDENTIFIER, 'cursor':STRING,
14
+ 'limit':{'type':'integer','minimum':1,'maximum':100}},()), effect='read')
15
+ async def feed(ctx, request, tx):
16
+ args = request.arguments
17
+ parent = await resolve(tx,args['parent']) if args.get('parent') else None
18
+ if parent:
19
+ await check_access(app,ctx,request,tx,parent,'list')
20
+ binding = digest({'parent':parent,'limit':args.get('limit',30)})
21
+ position = app.cursors.decode(args['cursor'],'rss',binding) if args.get('cursor') else None
22
+ parameters = []
23
+ conditions = []
24
+ if position is not None:
25
+ conditions.append(' AND (created_at,id)<(?,?)')
26
+ parameters.extend(position)
27
+ if parent:
28
+ conditions.append(' AND parent=?')
29
+ parameters.append(parent)
30
+ limit = args.get('limit',30)
31
+ items=[];more=False;last=position
32
+ for raw in tx.execute("SELECT body FROM resources WHERE type='post' AND state='active'"+
33
+ ''.join(conditions)+' ORDER BY created_at DESC,id DESC',parameters):
34
+ resource = decode(Resource,loads(raw[0]))
35
+ if not await visible(app,ctx,request,tx,resource.id):
36
+ continue
37
+ if len(items)==limit:
38
+ more=True
39
+ break
40
+ revision = await tx.revision(ResourceRef(id=resource.id))
41
+ items.append({'id':resource.id,'revision':revision.id,'title':resource.name,
42
+ 'url':app.settings.service_url+'/_id/'+resource.id,'created_at':wire(resource.created_at),
43
+ 'digest':revision.manifest_digest})
44
+ last=[wire(resource.created_at),resource.id]
45
+ data={'title':'msg.lmm.best','home':app.settings.service_url,'items':items}
46
+ if more:
47
+ cursor=app.cursors.encode('rss',binding,last)
48
+ data.update(cursor=cursor,next=next_link(app,'discovery.feed',{**args,'cursor':cursor}),
49
+ next_requires_auth=ctx.principal.subject is not None)
50
+ return HandlerOutput(data=data)
51
+
52
+
53
+ def render_feed(data):
54
+ from msg.core.codec import parse_time
55
+ root=ET.Element('rss',{'version':'2.0'})
56
+ channel=ET.SubElement(root,'channel')
57
+ for key,value in {'title':data['title'],'link':data['home'],
58
+ 'description':'Agent communication resource references','generator':'msg'}.items():
59
+ ET.SubElement(channel,key).text=value
60
+ for item in data['items']:
61
+ node=ET.SubElement(channel,'item')
62
+ ET.SubElement(node,'title').text=item['title']
63
+ ET.SubElement(node,'link').text=item['url']
64
+ ET.SubElement(node,'guid',{'isPermaLink':'false'}).text=item['id']
65
+ ET.SubElement(node,'pubDate').text=format_datetime(parse_time(item['created_at']),usegmt=True)
66
+ ET.SubElement(node,'description').text='Revision '+item['revision']+'; '+item['digest']
67
+ return ET.tostring(root,encoding='utf-8',xml_declaration=True)
msg/extensions/ssh.py ADDED
@@ -0,0 +1,264 @@
1
+ """OpenSSH forced-command adapter. No PTY, forwarding, shell or root proxy.
2
+
3
+ Only this OS-authenticated adapter constructs an SSH principal. A source field,
4
+ IP address, token, or network request can never opt into SSH authentication.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import asyncio
9
+ from dataclasses import replace
10
+ from datetime import timedelta
11
+ import os
12
+ from pathlib import Path
13
+ import re
14
+ import shlex
15
+ import sys
16
+ import time
17
+
18
+ from cryptography.hazmat.primitives import serialization
19
+ from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
20
+
21
+ from msg.constants import ROOT_SUBJECT
22
+ from msg.core.codec import canonical, decode, digest, wire
23
+ from msg.core.errors import Failure, require
24
+ from msg.core.packet import result_wire
25
+ from msg.core.models import Credential, Principal, Signature, SignatureProof, CapabilityGrant, HandlerOutput, EffectJob, ResourceRef
26
+ from msg.core.requests import request_for, payload_fields
27
+ from msg.plugins.common import operation_id, check_access, new_id, resolve
28
+ from msg.plugins.communication import event_id
29
+ from msg.plugins.identity import controlled_owner, validate_ceiling
30
+ from msg.plugins.schemas import obj, STRING, SIGNATURE, GRANTS, IDENTIFIER
31
+ from msg.security.crypto import key_id, verify
32
+
33
+
34
+ def public_key(line):
35
+ require(isinstance(line, str) and '\n' not in line and '\r' not in line, 'invalid_ssh_key')
36
+ fields = line.split()
37
+ require(len(fields) in {2, 3} and fields[0] == 'ssh-ed25519', 'unsupported_ssh_key')
38
+ normalized = ' '.join(fields[:2])
39
+ try:
40
+ key = serialization.load_ssh_public_key(normalized.encode('ascii'))
41
+ require(isinstance(key, Ed25519PublicKey), 'unsupported_ssh_key')
42
+ return normalized, key.public_bytes_raw()
43
+ except (ValueError, TypeError, UnicodeError) as exc:
44
+ raise Failure('invalid_ssh_key') from exc
45
+
46
+
47
+ def parse_command(command):
48
+ require(isinstance(command, str) and 0 < len(command) <= 32768 and
49
+ not any(ord(c) < 32 or ord(c) == 127 for c in command), 'ssh_command_denied')
50
+ try:
51
+ args = shlex.split(command, posix=True)
52
+ except ValueError as exc:
53
+ raise Failure('ssh_command_denied') from exc
54
+ if len(args) == 2 and args[0] in {'git-upload-pack', 'git-receive-pack'}:
55
+ require(re.fullmatch(r'/[@&][a-z][a-z0-9-]{1,40}/[A-Za-z0-9][A-Za-z0-9_.-]{0,90}\.git', args[1]) is not None,
56
+ 'ssh_command_denied')
57
+ return args[0], args[1]
58
+ if len(args) == 4 and args[:2] == ['msg', 'call']:
59
+ require(re.fullmatch(r'[a-z][a-z0-9_.]+', args[2]) is not None, 'ssh_command_denied')
60
+ return 'call', args[2], args[3]
61
+ if len(args) == 3 and args[:2] == ['msg', 'packet']:
62
+ require(re.fullmatch(r'[A-Za-z0-9_-]+', args[2]) is not None, 'ssh_command_denied')
63
+ return 'packet', args[2]
64
+ raise Failure('ssh_command_denied')
65
+
66
+
67
+ def require_sshd_process():
68
+ """Check an actual root-owned sshd ancestor, not a client-supplied environment."""
69
+ require(sys.platform == 'linux' and os.geteuid() != 0, 'ssh_os_isolation_required')
70
+ pid = os.getppid()
71
+ for _ in range(6):
72
+ if pid <= 1:
73
+ break
74
+ directory = Path('/proc') / str(pid)
75
+ try:
76
+ name = (directory / 'comm').read_text().strip()
77
+ status = dict(line.split(':', 1) for line in (directory / 'status').read_text().splitlines() if ':' in line)
78
+ if name in {'sshd', 'sshd-session', 'sshd-auth'}:
79
+ # Linux may hide /proc/<root-pid>/exe from an unprivileged
80
+ # child. The root-owned process identity cannot be forged by
81
+ # the dedicated service account; arbitrary ancestor names can.
82
+ uids = [int(value) for value in status['Uid'].split()]
83
+ if directory.stat().st_uid == 0 and len(uids) == 4 and all(uid == 0 for uid in uids):
84
+ return
85
+ # A root-owned executable proves nothing: an unprivileged
86
+ # Python/shell process can set its comm to "sshd". Continue to
87
+ # the privileged monitor/listener instead of trusting that
88
+ # process, including a user-launched OpenSSH daemon.
89
+ pid = int(status['PPid'].strip())
90
+ except (OSError, ValueError, KeyError):
91
+ break
92
+ raise Failure('ssh_os_isolation_required')
93
+
94
+
95
+ async def ssh_principal(app, credential_id, tx):
96
+ require(tx.setting('active_root_certificate',app.certificates.root_certificate.resource_id)==
97
+ app.certificates.root_certificate.resource_id,'service_restart_required')
98
+ credential = await tx.credential(credential_id)
99
+ require(credential.kind == 'ssh_key', 'wrong_credential_kind')
100
+ subject = await tx.subject(credential.subject_id)
101
+ require(subject.resource_id != ROOT_SUBJECT and not subject.local_only, 'local_only')
102
+ now = app.clock()
103
+ require(credential.revoked_at is None, 'credential_revoked')
104
+ require(credential.not_before <= now and (credential.expires_at is None or now < credential.expires_at),
105
+ 'credential_expired')
106
+ require(credential.ceiling, 'credential_ceiling')
107
+ certificates = tuple(tx.setting('ssh_certificates:' + credential_id, []))
108
+ principal = Principal(actor=subject.resource_id, subject=subject.resource_id, credential_id=credential.id,
109
+ method='ssh', certificates=certificates, ceiling=credential.ceiling)
110
+ await app.authorizer.grants(principal, tx)
111
+ return principal
112
+
113
+
114
+ class SSHAuthenticator:
115
+ """Bound to the credential OpenSSH has already authenticated, never wire data."""
116
+ def __init__(self, app, credential_id):
117
+ self.app, self.credential_id = app, credential_id
118
+
119
+ async def authenticate(self, request, session, *, entry):
120
+ require(entry == 'network', 'entry_not_allowed')
121
+ principal = await ssh_principal(self.app, self.credential_id, session)
122
+ spec = self.app.registry.operation(request.operation, request.contract_version)
123
+ require('network' in spec.entries, 'entry_not_allowed')
124
+ require(request.protocol_version == 1 and request.target_service == self.app.settings.service_url, 'wrong_service')
125
+ require(request.payload_digest == digest(payload_fields(request)), 'payload_digest_mismatch')
126
+ require(request.expires_at is not None and self.app.clock() < request.expires_at and
127
+ (request.expires_at - self.app.clock()).total_seconds() <= 300, 'request_expired')
128
+ require(request.subject in {None, principal.subject}, 'ssh_subject_mismatch')
129
+ require(any(f'{spec.name}@{spec.version}' in g.operations for g in principal.ceiling), 'credential_ceiling')
130
+ if request.proof is not None:
131
+ require(isinstance(request.proof, SignatureProof), 'ssh_signed_packet_required')
132
+ require(request.proof.signature.key_id == self.credential_id, 'ssh_signature_key_mismatch')
133
+ credential = await session.credential(self.credential_id)
134
+ from msg.core.requests import signing_bytes
135
+ verify(credential.verifier, signing_bytes(request), request.proof.signature, purpose='request')
136
+ # Bind the portable signature to the same SSH key. This preserves
137
+ # channel revocation for delayed jobs, rather than losing it by
138
+ # substituting another application key after the handshake.
139
+ for id in request.proof.certificates:
140
+ cert = await self.app.certificates.validate(id, session)
141
+ require(cert.key_id == credential.id and cert.subject_id == principal.subject,
142
+ 'certificate_subject_mismatch')
143
+ return replace(principal, method='signature', certificates=request.proof.certificates)
144
+ require(not spec.require_signature, 'signature_required')
145
+ # A successful SSH handshake is not a portable digitally signed ACK.
146
+ require(spec.name != 'discussion.ack', 'ack_signature_or_token_required')
147
+ return principal
148
+
149
+
150
+ def register(app, op):
151
+ @op('identity.ssh_key_add', obj({'public_key': STRING, 'proof': SIGNATURE, 'ceiling': GRANTS},
152
+ ('public_key', 'proof', 'ceiling')), signature=True)
153
+ async def add(ctx, request, tx):
154
+ subject = await controlled_owner(app, ctx, request, tx)
155
+ require(subject.kind == 'registered', 'registered_identity_required')
156
+ line, public = public_key(request.arguments['public_key'])
157
+ verify(public, canonical({'subject_id': subject.resource_id, 'public_key': line}),
158
+ decode(Signature, request.arguments['proof']), purpose='ssh-key-add')
159
+ grants = tuple(decode(CapabilityGrant, g) for g in request.arguments['ceiling'])
160
+ require(grants, 'credential_ceiling')
161
+ await validate_ceiling(app, ctx, tx, grants)
162
+ id = key_id(public)
163
+ require(tx.one('SELECT id FROM credentials WHERE id=?', (id,)) is None, 'credential_already_registered')
164
+ await tx.save_credential(Credential(id=id, subject_id=subject.resource_id, kind='ssh_key', verifier=public,
165
+ ceiling=grants, not_before=ctx.now, expires_at=None, revoked_at=None), subject.auth_version)
166
+ tx.set_setting('ssh_public:' + id, line)
167
+ return HandlerOutput(data={'key_id': id, 'subject_id': subject.resource_id})
168
+
169
+ @op('identity.ssh_key_revoke', obj({'key_id': IDENTIFIER}, ('key_id',)), signature=True)
170
+ async def revoke(ctx, request, tx):
171
+ subject = await controlled_owner(app, ctx, request, tx)
172
+ key = await tx.credential(request.arguments['key_id'])
173
+ require(key.kind == 'ssh_key' and key.subject_id == subject.resource_id, 'credential_subject_mismatch')
174
+ await tx.save_credential(replace(key, revoked_at=ctx.now), subject.auth_version)
175
+ return HandlerOutput(data={'key_id': key.id, 'revoked': True})
176
+
177
+ @op('identity.ssh_certificates', obj({'key_id': IDENTIFIER, 'certificates': {'type':'array','items':IDENTIFIER,'uniqueItems':True}},
178
+ ('key_id','certificates')), signature=True)
179
+ async def certificates(ctx, request, tx):
180
+ subject = await controlled_owner(app, ctx, request, tx)
181
+ key = await tx.credential(request.arguments['key_id'])
182
+ require(key.kind == 'ssh_key' and key.subject_id == subject.resource_id, 'credential_subject_mismatch')
183
+ for id in request.arguments['certificates']:
184
+ cert = await app.certificates.validate(id, tx)
185
+ require(cert.key_id == key.id and cert.subject_id == subject.resource_id, 'certificate_subject_mismatch')
186
+ tx.set_setting('ssh_certificates:' + key.id, request.arguments['certificates'])
187
+ return HandlerOutput(data={'key_id':key.id, 'certificates':request.arguments['certificates']})
188
+
189
+ @op('git.receive', obj({'id':IDENTIFIER}, ('id',)), effect='external')
190
+ async def receive(ctx, request, tx):
191
+ require(ctx.principal.method == 'ssh', 'ssh_transport_required')
192
+ rid = await resolve(tx, request.arguments['id'])
193
+ require((await tx.resource(rid)).type == 'repo', 'not_a_repository')
194
+ await check_access(app, ctx, request, tx, rid, 'write')
195
+ id, eid = new_id('job'), event_id(request, ctx.principal.subject)
196
+ job = EffectJob(id=id, event_id=eid, kind='git.receive', dedupe_key='git-receive:' + eid,
197
+ principal=ctx.principal, operation='git.receive', arguments={'id':rid, 'request_id':request.request_id},
198
+ state='running', attempts=1, next_attempt_at=ctx.now, lease_until=ctx.now+timedelta(seconds=600))
199
+ await tx.enqueue(job)
200
+ return HandlerOutput(resources=(ResourceRef(id=rid),), data={'job_id':id})
201
+
202
+
203
+ async def authorized_key(config_dir, key_type, key_body):
204
+ from msg.application import Application
205
+ from msg.config import load_settings
206
+ line, public = public_key(key_type + ' ' + key_body)
207
+ app = Application(load_settings(config_dir))
208
+ await app.load()
209
+ try:
210
+ async with app.metadata.transaction(write=False) as tx:
211
+ principal = await ssh_principal(app, key_id(public), tx)
212
+ require(tx.setting('ssh_public:' + principal.credential_id) == line, 'invalid_ssh_key')
213
+ command = shlex.join([sys.executable, '-m', 'msg.daemon', '--config-dir', str(Path(config_dir).resolve()),
214
+ 'ssh-session', '--credential', principal.credential_id])
215
+ escaped = command.replace('\\', '\\\\').replace('"', '\\"')
216
+ return f'restrict,command="{escaped}" {line}'
217
+ finally:
218
+ await app.close()
219
+
220
+
221
+ async def forced_session(config_dir, credential_id):
222
+ require_sshd_process()
223
+ from msg.application import Application
224
+ from msg.config import load_settings
225
+ from msg.daemon import network_runtime
226
+ app = Application(load_settings(config_dir))
227
+ network_runtime(app.settings)
228
+ await app.load()
229
+ try:
230
+ command = parse_command(os.environ.get('SSH_ORIGINAL_COMMAND', ''))
231
+ auth = SSHAuthenticator(app, credential_id)
232
+ executor = app.new_executor(auth)
233
+ async with app.metadata.transaction(write=False) as tx:
234
+ principal = await ssh_principal(app, credential_id, tx)
235
+ if command[0] in {'call','packet'}:
236
+ from msg.core.codec import loads
237
+ from msg.transports.packet import path_packet
238
+ if command[0] == 'packet':
239
+ packet = path_packet(command[1], 'j', app.settings.server.limits.max_request_bytes)
240
+ else:
241
+ packet = request_for(command[1], loads(command[2]), app.settings.service_url, subject=principal.subject,
242
+ expires_at=app.clock()+timedelta(seconds=120), source='manual')
243
+ result = await executor.execute(packet)
244
+ print(canonical(result_wire(result)).decode())
245
+ return 0 if result.status in {'ok','accepted'} else 1
246
+ from msg.extensions.repositories import NativeGitStore
247
+ store = NativeGitStore(app)
248
+ operation = 'git.refs' if command[0]=='git-upload-pack' else 'git.receive'
249
+ packet = request_for(operation, {'id':command[1]}, app.settings.service_url, subject=principal.subject,
250
+ expires_at=app.clock()+timedelta(seconds=120))
251
+ result = await executor.execute(packet)
252
+ require(result.status in {'ok','accepted'}, result.error.code if result.error else 'permission_denied')
253
+ rid = result.data['resource_id'] if operation=='git.refs' else result.resources[0].id
254
+ if operation=='git.receive':
255
+ from msg.extensions.ssh_git import receive_pack
256
+ return await receive_pack(app, result.data['job_id'])
257
+ process = await asyncio.create_subprocess_exec('git','--git-dir',str(store.path(rid)),
258
+ '-c','core.hooksPath=/dev/null','upload-pack','--strict',str(store.path(rid)),env=store.env)
259
+ try:
260
+ return await asyncio.wait_for(process.wait(),600)
261
+ finally:
262
+ if process.returncode is None:process.kill();await process.wait()
263
+ finally:
264
+ await app.close()
@@ -0,0 +1,300 @@
1
+ """Bridge Git's reference transaction to a current-authority metadata transaction.
2
+
3
+ Pack upload happens without holding a PostgreSQL write transaction. The
4
+ reference hook reauthorizes at ref preparation. Git and PostgreSQL are not one
5
+ atomic storage engine: a lost commit acknowledgement is recorded as uncertain.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ from dataclasses import replace
11
+ import hashlib
12
+ import hmac
13
+ import os
14
+ from pathlib import Path
15
+ import secrets
16
+ import shlex
17
+ import sys
18
+ import tempfile
19
+
20
+ from msg.core.codec import b64, canonical, loads, wire
21
+ from msg.core.errors import Failure, require
22
+ from msg.extensions.repositories import (NativeGitStore, MAX_GIT_PACK_BYTES, MAX_GIT_UPLOAD_SECONDS,
23
+ require_git_repository_capacity)
24
+ from msg.plugins.common import check_access
25
+ from msg.storage.git import durable_write
26
+ from msg.workers.effects import current_principal, worker_context, effect_request
27
+
28
+
29
+ class ReferenceGuard:
30
+ def __init__(self, app, job):
31
+ self.app, self.job = app, job
32
+ self.queue = asyncio.Queue()
33
+ self.changed = False
34
+ self.uncertain = False
35
+ self.failures = []
36
+
37
+ async def authorize(self, tx):
38
+ current = await tx.job(self.job.id)
39
+ require(current.state == 'running' and current.attempts == self.job.attempts, 'job_lease_lost')
40
+ principal = await current_principal(self.app, self.job.principal, tx)
41
+ context, request = worker_context(self.app, self.job, principal), effect_request(self.app, self.job, principal)
42
+ await check_access(self.app, context, request, tx, self.job.arguments['id'], 'write')
43
+ require(tx.setting('runtime_config', {}).get('accept_writes', True), 'writes_paused')
44
+ return principal
45
+
46
+ @staticmethod
47
+ def validate_changes(text):
48
+ require(isinstance(text, str) and len(text.encode()) <= 65536, 'git_ref_batch_too_large')
49
+ rows = []
50
+ import re
51
+ for line in text.splitlines():
52
+ fields = line.split(' ')
53
+ require(len(fields) == 3, 'invalid_git_change')
54
+ old, new, ref = fields
55
+ require(re.fullmatch(r'[0-9a-f]{40}', old) is not None and re.fullmatch(r'[0-9a-f]{40}', new) is not None,
56
+ 'invalid_git_change')
57
+ require(ref == 'HEAD' or re.fullmatch(r'refs/(heads|tags)/[^\s]+', ref) is not None, 'invalid_git_ref')
58
+ rows.append({'ref':ref, 'old':old, 'new':new})
59
+ require(0 < len(rows) <= 128 and len({r['ref'] for r in rows}) == len(rows), 'invalid_git_change')
60
+ return rows
61
+
62
+ async def run(self):
63
+ """One task owns both halves of each PostgreSQL transaction."""
64
+ manager = tx = None
65
+ preparing = prepared = None
66
+ while True:
67
+ message, answer = await self.queue.get()
68
+ try:
69
+ state = message['state']
70
+ if state == 'stop':
71
+ if manager is not None:
72
+ self.uncertain = True
73
+ await manager.__aexit__(Failure, Failure('external_uncertain'), None)
74
+ answer.set_result({'ok':True})
75
+ return
76
+ changes = self.validate_changes(message['changes'])
77
+ if state == 'preparing':
78
+ require(manager is None, 'git_transaction_already_prepared')
79
+ # Git 2.54+ invokes this phase before taking ref locks.
80
+ # Preflight current authority without holding a PostgreSQL write
81
+ # transaction, then re-check inside the prepared phase.
82
+ async with self.app.metadata.transaction(write=False) as preflight:
83
+ await self.authorize(preflight)
84
+ preparing = changes
85
+ elif state == 'prepared':
86
+ require(manager is None, 'git_transaction_already_prepared')
87
+ require(preparing is None or changes == preparing, 'git_transaction_mismatch')
88
+ manager = self.app.metadata.transaction(write=True)
89
+ tx = await manager.__aenter__()
90
+ try:
91
+ await self.authorize(tx)
92
+ except BaseException:
93
+ await manager.__aexit__(*sys.exc_info())
94
+ manager = tx = None
95
+ raise
96
+ prepared = changes
97
+ elif state in {'committed', 'aborted'}:
98
+ require(manager is not None and changes == prepared, 'git_transaction_mismatch')
99
+ active = manager
100
+ manager = None
101
+ if state == 'aborted':
102
+ await active.__aexit__(Failure, Failure('git_transaction_aborted'), None)
103
+ else:
104
+ self.changed = True
105
+ try:
106
+ resource = await tx.resource(self.job.arguments['id'])
107
+ await tx.replace(replace(resource, generation=resource.generation + 1,
108
+ modified_at=self.app.clock(), modified_by=self.job.principal.actor), resource.generation)
109
+ history = tx.setting('git_receive:' + self.job.id, [])
110
+ history.append({'time':wire(self.app.clock()), 'refs':changes})
111
+ tx.set_setting('git_receive:' + self.job.id, history)
112
+ from msg.core.models import Event, AuditEvent, ResourceRef
113
+ from msg.core.codec import digest
114
+ from msg.plugins.common import new_id
115
+ event = Event(id=new_id('event'), type='git.refs_committed', time=self.app.clock(),
116
+ request_id=self.job.arguments['request_id'], actor=self.job.principal.actor,
117
+ subject=self.job.principal.subject, resources=(), data={'job_id':self.job.id, 'refs':changes})
118
+ await tx.append_event(event)
119
+ await tx.append_audit(AuditEvent(event=event,
120
+ authority=tuple(ResourceRef(id=c) for c in self.job.principal.certificates),
121
+ before_digest=digest(resource), after_digest=digest(changes),
122
+ previous_digest=None, entry_digest='', result='refs_committed'))
123
+ await active.__aexit__(None, None, None)
124
+ except BaseException:
125
+ self.uncertain = True
126
+ if not tx.closed:
127
+ await active.__aexit__(*sys.exc_info())
128
+ raise
129
+ tx = preparing = prepared = None
130
+ else:
131
+ raise Failure('invalid_git_hook_state')
132
+ answer.set_result({'ok':True})
133
+ except BaseException as exc:
134
+ code = exc.code if isinstance(exc, Failure) else 'git_guard_failed'
135
+ self.failures.append(code)
136
+ if not answer.done():
137
+ answer.set_result({'ok':False, 'code':code})
138
+
139
+ async def message(self, value):
140
+ answer = asyncio.get_running_loop().create_future()
141
+ await self.queue.put((value, answer))
142
+ return await asyncio.shield(answer)
143
+
144
+
145
+ def hook_program(socket_path, secret):
146
+ """Generated from trusted code, not from a repository or user command."""
147
+ return ('#!' + sys.executable + '\n' +
148
+ 'import json,socket,sys\n'
149
+ 'data=sys.stdin.buffer.read(65537)\n'
150
+ 'if len(data)>65536: sys.exit(1)\n'
151
+ 's=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM);s.settimeout(45)\n' +
152
+ 's.connect(' + repr(str(socket_path)) + ')\n' +
153
+ 's.sendall((json.dumps({"secret":' + repr(secret) + ',"state":sys.argv[1],"changes":data.decode("ascii")})+"\\n").encode())\n'
154
+ 'r=s.makefile("rb").readline(8193)\n'
155
+ 'sys.exit(0 if r and len(r)<=8192 and json.loads(r).get("ok") else 1)\n').encode()
156
+
157
+
158
+ async def relay_bounded_stdin(process, stream, *, limit, timeout):
159
+ """Copy stream to process stdin, rejecting once limit bytes have been seen."""
160
+ loop=asyncio.get_running_loop()
161
+ total=0
162
+ async def pump():
163
+ nonlocal total
164
+ while True:
165
+ chunk=await loop.run_in_executor(None, stream.read, 65536)
166
+ if not chunk:
167
+ break
168
+ total+=len(chunk)
169
+ require(total<=limit,'request_too_large')
170
+ process.stdin.write(chunk)
171
+ await process.stdin.drain()
172
+ process.stdin.close()
173
+ await asyncio.wait_for(pump(), timeout)
174
+ return await asyncio.wait_for(process.wait(), timeout)
175
+
176
+
177
+ async def guarded_command(app, job, command_factory, *, stdin=None, stdout=None, stderr=None, timeout=600,
178
+ input_data=None,input_file=None,capture_output=False,output_limit=None,cache_result_key=None,
179
+ stdin_byte_limit=None,stdin_stream=None):
180
+ """Run a fixed Git command with a private reference-transaction hook.
181
+
182
+ command_factory is installed adapter code, never a value accepted from wire.
183
+ Tests exercise this bridge with real update-ref; the SSH adapter uses receive-pack.
184
+ """
185
+ store = NativeGitStore(app)
186
+ require(input_data is None or input_file is None,'ambiguous_git_input')
187
+ require(stdin_byte_limit is None or (input_data is None and input_file is None),'ambiguous_git_input')
188
+ app.settings.server.staging_dir.mkdir(parents=True, exist_ok=True)
189
+ # Unix socket path has a small OS limit; the random private directory is not
190
+ # an account directory and contains no application credentials.
191
+ with tempfile.TemporaryDirectory(prefix='msg-git-') as temporary:
192
+ directory = Path(temporary)
193
+ socket_path = directory/'guard.sock'
194
+ secret = secrets.token_hex(32)
195
+ guard = ReferenceGuard(app, job)
196
+ task = asyncio.create_task(guard.run())
197
+ async def connection(reader, writer):
198
+ try:
199
+ raw = await asyncio.wait_for(reader.readline(), 45)
200
+ require(len(raw) <= 70000, 'git_hook_message_too_large')
201
+ value = loads(raw)
202
+ require(hmac.compare_digest(value.get('secret',''), secret), 'git_hook_authentication_failed')
203
+ result = await guard.message(value)
204
+ writer.write(canonical(result) + b'\n')
205
+ await writer.drain()
206
+ except Exception:
207
+ writer.write(b'{"ok":false}\n')
208
+ try:await writer.drain()
209
+ except (BrokenPipeError, ConnectionError):pass
210
+ finally:
211
+ writer.close()
212
+ await writer.wait_closed()
213
+ server = await asyncio.start_unix_server(connection, str(socket_path), limit=70001)
214
+ durable_write(directory/'reference-transaction', hook_program(socket_path, secret), mode=0o700)
215
+ env = dict(store.env)
216
+ command = ['git','--git-dir',str(store.path(job.arguments['id'])),'-c','core.fsync=all',
217
+ '-c','core.hooksPath='+str(directory),'-c','receive.fsckObjects=true',
218
+ '-c','receive.denyNonFastForwards=true',*command_factory(store)]
219
+ process = None
220
+ code = None
221
+ output = b''
222
+ try:
223
+ process = await asyncio.create_subprocess_exec(*command,
224
+ stdin=asyncio.subprocess.PIPE if input_data is not None or input_file is not None or stdin_byte_limit is not None else stdin,
225
+ stdout=asyncio.subprocess.PIPE if capture_output else stdout,
226
+ stderr=asyncio.subprocess.DEVNULL if capture_output else stderr,
227
+ env=env,start_new_session=True)
228
+ if input_data is not None or input_file is not None:
229
+ async def exchange():
230
+ async def feed():
231
+ try:
232
+ if input_file is None:
233
+ process.stdin.write(input_data)
234
+ await process.stdin.drain()
235
+ else:
236
+ with open(input_file,'rb') as source:
237
+ while chunk:=source.read(65536):
238
+ process.stdin.write(chunk)
239
+ await process.stdin.drain()
240
+ except (BrokenPipeError,ConnectionResetError):
241
+ pass
242
+ finally:
243
+ process.stdin.close()
244
+ sender=asyncio.create_task(feed())
245
+ try:
246
+ chunks=[]
247
+ size=0
248
+ while chunk:=await process.stdout.read(65536):
249
+ size+=len(chunk)
250
+ require(output_limit is None or size<=output_limit,'response_too_large')
251
+ chunks.append(chunk)
252
+ await sender
253
+ await process.wait()
254
+ return b''.join(chunks)
255
+ finally:
256
+ if not sender.done():sender.cancel()
257
+ await asyncio.gather(sender,return_exceptions=True)
258
+ output=await asyncio.wait_for(exchange(),timeout)
259
+ code=process.returncode
260
+ else:
261
+ if stdin_byte_limit is not None:
262
+ code = await relay_bounded_stdin(process, sys.stdin.buffer if stdin_stream is None else stdin_stream,
263
+ limit=stdin_byte_limit, timeout=timeout)
264
+ else:
265
+ code = await asyncio.wait_for(process.wait(), timeout)
266
+ finally:
267
+ if process is not None and process.returncode is None:
268
+ import signal
269
+ os.killpg(process.pid, signal.SIGKILL)
270
+ await process.wait()
271
+ guard.uncertain = True
272
+ await guard.message({'state':'stop'})
273
+ await task
274
+ server.close()
275
+ await server.wait_closed()
276
+ async with app.metadata.transaction(write=True) as tx:
277
+ current = await tx.job(job.id)
278
+ if current.state == 'running':
279
+ # Git ref transactions may have partially succeeded without
280
+ # --atomic. Audit the actual committed refs, never invent rollback.
281
+ uncertain=guard.uncertain or (guard.changed and (code != 0 or bool(guard.failures)))
282
+ state = 'uncertain' if uncertain else 'done' if code == 0 and not guard.failures else 'failed'
283
+ if state=='done' and cache_result_key is not None:
284
+ require(capture_output,'git_result_cache_requires_output')
285
+ tx.set_setting(cache_result_key,b64(output))
286
+ await tx.save_job(replace(current, state=state, lease_until=None))
287
+ tx.set_setting('job_status:'+job.id, {'code':'external_uncertain' if uncertain else
288
+ guard.failures[0] if guard.failures else 'ok' if state=='done' else 'git_operation_failed',
289
+ 'refs_changed':guard.changed})
290
+ result=code if code is not None and not guard.uncertain and not guard.failures else 1
291
+ return (result,output,guard.changed) if capture_output else result
292
+
293
+
294
+ async def receive_pack(app, job_id, *, stdin_stream=None):
295
+ async with app.metadata.transaction(write=False) as tx:
296
+ job = await tx.job(job_id)
297
+ require(job.kind == 'git.receive' and job.state == 'running', 'invalid_git_session')
298
+ require_git_repository_capacity(NativeGitStore(app).path(job.arguments['id']))
299
+ return await guarded_command(app, job, lambda store:['receive-pack',str(store.path(job.arguments['id']))],
300
+ timeout=MAX_GIT_UPLOAD_SECONDS, stdin_byte_limit=MAX_GIT_PACK_BYTES, stdin_stream=stdin_stream)