msgctl 0.1.0a1__py3-none-any.whl

msg/security/vault.py

@@ -0,0 +1,186 @@
+"""Domain-separated AES-GCM storage for server-held custodial key material."""
+from __future__ import annotations
+
+import os
+import hmac
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives import hashes
+
+from msg.core.codec import b64,canonical,unb64,wire
+from msg.core.errors import Failure,require
+from msg.security.age_keys import recipient_from_identity,private_from_identity
+from msg.security.crypto import Ed25519Signer
+
+
+def _aad(subject,key_id,kind):
+    return canonical({'domain':'msg-custodial-vault-v1','subject':subject,
+                      'key_id':key_id,'kind':kind})
+
+
+def _seal(app,subject,key_id,kind,secret):
+    nonce=os.urandom(12)
+    return b64(nonce),b64(AESGCM(app._vault_key).encrypt(nonce,secret,
+                                                       _aad(subject,key_id,kind)))
+
+
+def store_keys(app,tx,subject,signer,age_identity,age_key_id,now):
+    require(recipient_from_identity(age_identity),'invalid_age_identity')
+    signing_nonce,signing_ciphertext=_seal(app,subject,signer.key_id,'identity',
+                                           signer.private_bytes())
+    age_nonce,age_ciphertext=_seal(app,subject,age_key_id,'encryption',
+                                   age_identity.encode('ascii'))
+    tx.execute('''INSERT INTO custodial_vault
+        (subject,signing_key_id,encryption_key_id,signing_nonce,signing_ciphertext,
+         age_nonce,age_ciphertext,status,created_at,destroyed_at)
+        VALUES (?,?,?,?,?,?,?,?,?,?)''',
+        (subject,signer.key_id,age_key_id,signing_nonce,signing_ciphertext,
+         age_nonce,age_ciphertext,'active',wire(now),None),write=True)
+
+
+def _open(app,tx,subject,kind):
+    row=tx.one('''SELECT signing_key_id,encryption_key_id,signing_nonce,signing_ciphertext,
+        age_nonce,age_ciphertext,status FROM custodial_vault WHERE subject=?''',(subject,))
+    require(row is not None and (row[6]=='active' or
+            (kind=='encryption' and row[6]=='decrypt_only')),'custodial_vault_unavailable')
+    key_id,nonce,ciphertext=(row[0],row[2],row[3]) if kind=='identity' else (row[1],row[4],row[5])
+    require(nonce is not None and ciphertext is not None,'custodial_vault_unavailable')
+    try:
+        return key_id,AESGCM(app._vault_key).decrypt(unb64(nonce,limit=12),
+            unb64(ciphertext,limit=8192),_aad(subject,key_id,kind))
+    except (InvalidTag,ValueError,TypeError) as exc:
+        raise Failure('custodial_vault_corrupt') from exc
+
+
+def open_signer(app,tx,subject):
+    key_id,secret=_open(app,tx,subject,'identity')
+    signer=Ed25519Signer.from_bytes(secret)
+    require(signer.key_id==key_id,'custodial_vault_corrupt')
+    return signer
+
+
+def open_age_identity(app,tx,subject):
+    key_id,secret=_open(app,tx,subject,'encryption')
+    try:
+        identity=secret.decode('ascii')
+    except UnicodeError as exc:
+        raise Failure('custodial_vault_corrupt') from exc
+    from msg.security.age_keys import encryption_key_id,public_from_recipient
+    recipient=recipient_from_identity(identity)
+    require(encryption_key_id(public_from_recipient(recipient))==key_id,
+            'custodial_vault_corrupt')
+    return identity
+
+
+def rewrap_owned_age_ciphertext(identity,recipient,ciphertext):
+    """Bounded age-to-age conversion; never expose the decrypted bytes to a caller."""
+    from msg.security.age_keys import public_from_recipient
+    public_from_recipient(recipient)
+    require(type(ciphertext) is bytes and
+            ciphertext.startswith(b'age-encryption.org/v1\n') and
+            len(ciphertext)<=1048576,'custodial_rewrap_ciphertext_invalid')
+    executable=shutil.which('age')
+    require(executable is not None,'age_dependency_unavailable')
+    with tempfile.TemporaryDirectory(prefix='msg-custodial-rewrap-') as folder:
+        secret=Path(folder)/'identity'
+        secret.write_bytes((identity+'\n').encode('ascii'))
+        secret.chmod(0o600)
+        try:
+            opened=subprocess.run([executable,'--decrypt','--identity',str(secret)],
+                input=ciphertext,capture_output=True,timeout=15,check=False)
+            require(opened.returncode==0 and len(opened.stdout)<=1048576,
+                    'custodial_rewrap_decrypt_failed')
+            sealed=subprocess.run([executable,'--encrypt','--recipient',recipient],
+                input=opened.stdout,capture_output=True,timeout=15,check=False)
+            require(sealed.returncode==0 and
+                    sealed.stdout.startswith(b'age-encryption.org/v1\n') and
+                    len(sealed.stdout)<=1114112,'custodial_rewrap_encrypt_failed')
+        except (OSError,subprocess.TimeoutExpired) as exc:
+            raise Failure('custodial_rewrap_operation_failed') from exc
+        return sealed.stdout
+
+
+def _upgrade_aad(subject,challenge_id):
+    return canonical({'domain':'msg-custodial-upgrade-ephemeral-v1',
+                      'subject':subject,'challenge_id':challenge_id})
+
+
+def seal_upgrade_private(app,subject,challenge_id,private):
+    nonce=os.urandom(12)
+    cipher=AESGCM(app._vault_key).encrypt(nonce,private,
+                                           _upgrade_aad(subject,challenge_id))
+    return b64(nonce),b64(cipher)
+
+
+def open_upgrade_private(app,subject,challenge_id,nonce,ciphertext):
+    try:
+        raw=AESGCM(app._vault_key).decrypt(unb64(nonce,limit=12),
+            unb64(ciphertext,limit=128),_upgrade_aad(subject,challenge_id))
+        return X25519PrivateKey.from_private_bytes(raw)
+    except (InvalidTag,ValueError,TypeError) as exc:
+        raise Failure('custodial_upgrade_challenge_corrupt') from exc
+
+
+UPGRADE_CONTEXT_FIELDS=('subject_id','challenge_id','request_id','handle','public_key',
+                        'encryption_recipient','server_public','nonce','expires_at')
+
+
+def _proof(shared,challenge):
+    context={name:challenge[name] for name in UPGRADE_CONTEXT_FIELDS}
+    key=HKDF(algorithm=hashes.SHA256(),length=32,
+             salt=unb64(context['nonce'],limit=64),
+             info=b'msg-custodial-upgrade-pop-v1').derive(shared)
+    return b64(hmac.digest(key,canonical(context),'sha256'))
+
+
+def client_upgrade_proof(age_identity,challenge):
+    private=private_from_identity(age_identity)
+    shared=private.exchange(X25519PublicKey.from_public_bytes(
+        unb64(challenge['server_public'],limit=32)))
+    return _proof(shared,challenge)
+
+
+def server_upgrade_proof(app,subject,challenge,nonce,ciphertext):
+    from msg.security.age_keys import public_from_recipient
+    private=open_upgrade_private(app,subject,challenge['challenge_id'],nonce,ciphertext)
+    shared=private.exchange(X25519PublicKey.from_public_bytes(
+        public_from_recipient(challenge['encryption_recipient'])))
+    return _proof(shared,challenge)
+
+
+def seal_retired_encryption_key(app, tx, subject, old_key_id, new_recipient, challenge_id):
+    """Export only this subject's retired age subkey, encrypted to its proved new key."""
+    from msg.security.age_keys import encryption_key_id, public_from_recipient
+    row = tx.one('SELECT subject,recipient,retired_at,is_primary FROM encryption_subkeys WHERE key_id=?',
+                 (old_key_id,))
+    require(row is not None and row[0] == subject and row[2] is not None and row[3] == 0,
+            'custodial_recovery_requires_retired_owned_key')
+    vault = tx.one('SELECT encryption_key_id FROM custodial_vault WHERE subject=?', (subject,))
+    require(vault == (old_key_id,), 'custodial_recovery_key_mismatch')
+    primary = tx.one('SELECT key_id,recipient FROM encryption_subkeys WHERE subject=? AND is_primary=1',
+                     (subject,))
+    new_key_id = encryption_key_id(public_from_recipient(new_recipient))
+    require(primary == (new_key_id, new_recipient) and new_key_id != old_key_id,
+            'custodial_recovery_recipient_mismatch')
+    identity = open_age_identity(app, tx, subject)
+    payload = {'format': 'msg-custodial-retired-key-v1',
+               'purpose': 'retired-encryption-subkey-recovery', 'subject_id': subject,
+               'challenge_id': challenge_id, 'encryption_key_id': old_key_id,
+               'encryption_recipient': row[1], 'age_identity': identity}
+    executable = shutil.which('age')
+    require(executable is not None, 'age_dependency_unavailable')
+    try:
+        result = subprocess.run([executable, '--encrypt', '--recipient', new_recipient],
+            input=canonical(payload), capture_output=True, timeout=15, check=False)
+    except (OSError, subprocess.TimeoutExpired):
+        raise Failure('custodial_recovery_encryption_failed') from None
+    require(result.returncode == 0 and result.stdout.startswith(b'age-encryption.org/v1\n'),
+            'custodial_recovery_encryption_failed')
+    return result.stdout

msg/storage/capacity.py

@@ -0,0 +1,100 @@
+"""Deployment-wide durability caps for tables that otherwise grow with fresh identities.
+
+Personal entitlements stay elsewhere. These checks only stop one shared database
+or content volume from being filled by unbounded metadata, staged bytes, or
+ciphertext. They do not grant quota, delete user content, or clear quarantine.
+"""
+from __future__ import annotations
+
+from msg.core.codec import decode, loads
+from msg.core.errors import Failure, require
+from msg.core.models import Revision
+
+MAX_TRANSFERS = 4096
+MAX_STAGED_CHUNK_BYTES = 4 * 1024 * 1024 * 1024
+MAX_PURGE_RECORDS = 4096
+MAX_PENDING_CSRS = 1024
+MAX_CSR_ROWS = 8192
+MAX_QUEUED_WEBHOOKS = 1024
+MAX_WEBHOOK_ROWS = 8192
+MAX_RESULTS = 100_000
+MAX_REACTIONS = 100_000
+MAX_KEYSTORE_REVISIONS = 4096
+MAX_KEYSTORE_BYTES = 64 * 1024 * 1024
+_CAPACITY = 'storage_capacity_exceeded'
+
+
+def _int(value):
+    if type(value) is bool:
+        raise Failure(_CAPACITY)
+    try:
+        number = int(value)
+    except (TypeError, ValueError, OverflowError) as exc:
+        raise Failure(_CAPACITY) from exc
+    require(number >= 0 and number == value, _CAPACITY)
+    return number
+
+
+def _count(tx, sql, args=()):
+    return _int(tx.one(sql, args)[0])
+
+
+def require_transfer_capacity(tx, extra_bytes=0, *, new_transfer=False):
+    require(type(extra_bytes) is int and extra_bytes >= 0, _CAPACITY)
+    if new_transfer:
+        require(_count(tx, 'SELECT COUNT(*) FROM transfers') < MAX_TRANSFERS, _CAPACITY)
+    staged = _count(tx, 'SELECT COALESCE(SUM(length), 0) FROM chunks')
+    require(staged + extra_bytes <= MAX_STAGED_CHUNK_BYTES, _CAPACITY)
+
+
+def trim_purge_records(tx):
+    rows = tx.rows("SELECT key, value FROM settings WHERE key LIKE 'purge_record:%'")
+    if len(rows) <= MAX_PURGE_RECORDS:
+        return
+    def stamp(raw):
+        value = loads(raw)
+        require(isinstance(value, dict), _CAPACITY)
+        return str(value.get('time') or '')
+    ranked = sorted(rows, key=lambda row: (stamp(row[1]), row[0]))
+    for key, _value in ranked[:len(ranked) - MAX_PURGE_RECORDS]:
+        tx.execute('DELETE FROM settings WHERE key=?', (key,), write=True)
+
+
+def require_csr_capacity(tx):
+    require(_count(tx, "SELECT COUNT(*) FROM csrs WHERE state='pending'") < MAX_PENDING_CSRS, _CAPACITY)
+    require(_count(tx, 'SELECT COUNT(*) FROM csrs') < MAX_CSR_ROWS, _CAPACITY)
+
+
+def require_webhook_capacity(tx):
+    active = _count(tx, "SELECT COUNT(*) FROM jobs WHERE kind='webhook' AND state IN ('pending','running')")
+    require(active < MAX_QUEUED_WEBHOOKS, _CAPACITY)
+    total = _count(tx, "SELECT COUNT(*) FROM jobs WHERE kind='webhook'")
+    if total < MAX_WEBHOOK_ROWS:
+        return
+    overflow = total - MAX_WEBHOOK_ROWS + 1
+    finished = tx.rows(
+        "SELECT id FROM jobs WHERE kind='webhook' AND state IN ('done','failed') ORDER BY next_at, id LIMIT ?",
+        (overflow,))
+    for (job_id,) in finished:
+        tx.execute("DELETE FROM jobs WHERE id=? AND kind='webhook' AND state IN ('done','failed')",
+                   (job_id,), write=True)
+    require(_count(tx, "SELECT COUNT(*) FROM jobs WHERE kind='webhook'") < MAX_WEBHOOK_ROWS, _CAPACITY)
+
+
+def require_result_capacity(tx):
+    require(_count(tx, 'SELECT COUNT(*) FROM results') < MAX_RESULTS, _CAPACITY)
+
+
+def require_reaction_capacity(tx):
+    require(_count(tx, 'SELECT COUNT(*) FROM reactions') < MAX_REACTIONS, _CAPACITY)
+
+
+def require_keystore_capacity(tx, extra_bytes):
+    require(type(extra_bytes) is int and extra_bytes >= 0, _CAPACITY)
+    require(_count(tx, """SELECT COUNT(*) FROM revisions r JOIN resources s
+        ON s.id=r.resource_id WHERE s.type='keystore'""") < MAX_KEYSTORE_REVISIONS, _CAPACITY)
+    total = 0
+    for (raw,) in tx.execute("""SELECT r.body FROM revisions r JOIN resources s
+            ON s.id=r.resource_id WHERE s.type='keystore'"""):
+        total += _int(decode(Revision, loads(raw)).content.size)
+        require(total + extra_bytes <= MAX_KEYSTORE_BYTES, _CAPACITY)

msg/storage/custodial_migration.py

@@ -0,0 +1,22 @@
+"""One idempotent PostgreSQL upgrade for the decrypt-only vault lifecycle."""
+from msg.core.errors import require
+
+LEGACY_STATUS = "CHECK ((status = ANY (ARRAY['active'::text, 'destroyed'::text])))"
+LIFECYCLE_CHECK = """CHECK (
+    status IN ('active','decrypt_only','destroyed') AND
+    (status <> 'decrypt_only' OR (signing_nonce IS NULL AND signing_ciphertext IS NULL
+        AND age_nonce IS NOT NULL AND age_ciphertext IS NOT NULL)))"""
+
+
+def migrate_custodial_vault(connection):
+    """Caller holds the schema advisory lock; neither keys nor rows are rewritten."""
+    old = connection.execute("""SELECT pg_get_constraintdef(oid) FROM pg_constraint
+        WHERE conrelid='custodial_vault'::regclass AND conname='custodial_vault_status_check'""").fetchone()
+    if old is not None:
+        require(old[0] == LEGACY_STATUS, 'custodial_vault_unknown_constraint')
+        connection.execute('ALTER TABLE custodial_vault DROP CONSTRAINT custodial_vault_status_check')
+    present = connection.execute("""SELECT 1 FROM pg_constraint WHERE conrelid='custodial_vault'::regclass
+        AND conname='custodial_vault_lifecycle_check'""").fetchone()
+    if present is None:
+        connection.execute('ALTER TABLE custodial_vault ADD CONSTRAINT custodial_vault_lifecycle_check '
+                           + LIFECYCLE_CHECK)