msgctl 0.1.0a1__py3-none-any.whl

msg/core/text_patch.py

@@ -0,0 +1,283 @@
+"""Bounded, byte-preserving text edits. No fuzzy matching or executable syntax.
+
+A heading edit replaces its complete Markdown section, including its heading.
+A block is a heading, fenced code block, or contiguous nonblank text. Empty
+separator lines are not included. Digests cover the selected original UTF-8.
+Unified patches describe one file; their filenames never select a resource.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import re
+
+from msg.core.codec import digest, wire
+from msg.core.errors import require
+
+PATCH_LIMIT = 1048576
+PATCH_CONTEXT_LIMIT = 256
+PATCH_CANDIDATE_LIMIT = 4096
+MAX_HUNKS = 256
+MAX_LINES = 65536
+
+_TEXT = {'type':'string', 'maxLength':PATCH_LIMIT}
+_DIGEST = {'type':'string', 'pattern':r'^sha256:[0-9a-f]{64}$'}
+
+
+def _variant(kind, properties, required):
+    return {'type':'object', 'properties':{'kind':{'const':kind}, **properties},
+            'required':['kind', *required], 'additionalProperties':False}
+
+
+PATCH_SCHEMA = {'oneOf':[
+    _variant('exact', {'exact':{**_TEXT, 'minLength':1}, 'replacement':_TEXT,
+        'before':{'type':'string','maxLength':PATCH_CONTEXT_LIMIT},
+        'after':{'type':'string','maxLength':PATCH_CONTEXT_LIMIT}}, ('exact','replacement')),
+    _variant('unified', {'diff':{**_TEXT, 'minLength':1}}, ('diff',)),
+    _variant('heading', {'heading':{'type':'string','minLength':1,'maxLength':256},
+        'digest':_DIGEST, 'replacement':_TEXT}, ('heading','digest','replacement')),
+    _variant('block', {'digest':_DIGEST, 'replacement':_TEXT}, ('digest','replacement')),
+]}
+
+
+def validate_patch(patch):
+    # The public registry validates the same closed schema. Keep this guard for
+    # direct callers and batch preparation, before any content is persisted.
+    from jsonschema import Draft202012Validator
+    require(Draft202012Validator(PATCH_SCHEMA).is_valid(wire(patch)), 'invalid_text_patch')
+    require(sum(len(value.encode('utf-8')) for value in patch.values()
+                if isinstance(value,str)) <= PATCH_LIMIT, 'patch_too_large')
+
+
+def apply_text_patch(source, exact, replacement, before='', after=''):
+    """Legacy contextual replacement, with unchanged ambiguity semantics."""
+    require(bool(exact), 'patch_exact_required')
+    match = None
+    start = candidates = 0
+    while True:
+        index = source.find(exact, start)
+        if index < 0:
+            break
+        candidates += 1
+        require(candidates <= PATCH_CANDIDATE_LIMIT, 'patch_too_complex')
+        end = index + len(exact)
+        if (index >= len(before) and source.startswith(before,index-len(before))
+                and source.startswith(after,end)):
+            require(match is None, 'patch_ambiguous')
+            match = index
+        start = index + 1
+    require(match is not None, 'patch_no_match')
+    result = source[:match] + replacement + source[match+len(exact):]
+    require(len(result.encode('utf-8')) <= PATCH_LIMIT, 'patch_too_large')
+    return result
+
+
+@dataclass(frozen=True)
+class Hunk:
+    old_start: int
+    new_start: int
+    old: tuple[str, ...]
+    new: tuple[str, ...]
+
+
+_HUNK = re.compile(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(?:[^\r\n]*)\r?\n?\Z')
+_NO_NEWLINE = '\'
+
+
+def _diff_lines(text):
+    # Git/unified diff separates lines only at LF. str.splitlines also splits
+    # Unicode separators and CR, changing the signed bytes and hunk counts.
+    return re.findall(r'[^\n]*\n|[^\n]+$', text)
+
+
+def _hunks(text):
+    lines = _diff_lines(text)
+    require(len(lines) <= MAX_LINES, 'patch_too_complex')
+    i = 0
+    if lines and lines[0].startswith('--- '):
+        require(len(lines)>1 and lines[1].startswith('+++ '), 'invalid_unified_patch')
+        i = 2
+    hunks = []
+    while i < len(lines):
+        match = _HUNK.fullmatch(lines[i])
+        require(match is not None and len(hunks)<MAX_HUNKS, 'invalid_unified_patch')
+        old_no, old_count, new_no, new_count = (
+            int(match[1]), int(match[2] or 1), int(match[3]), int(match[4] or 1))
+        require((not old_count or old_no>0) and (not new_count or new_no>0),
+                'invalid_unified_patch')
+        i += 1
+        old, new = [], []
+        previous = None
+        while i < len(lines) and not lines[i].startswith('@@ '):
+            line = lines[i]
+            if line.rstrip('\r\n') == _NO_NEWLINE:
+                require(previous is not None, 'invalid_unified_patch')
+                for side in previous:
+                    require(side[-1].endswith('\n'), 'invalid_unified_patch')
+                    side[-1] = side[-1][:-1]
+                previous = None
+            else:
+                require(line[:1] in {' ','-','+'}, 'invalid_unified_patch')
+                # Once counts are exhausted, another file header is not a hunk.
+                require(len(old)<old_count or len(new)<new_count, 'invalid_unified_patch')
+                previous = []
+                if line[0] in ' -':
+                    old.append(line[1:]); previous.append(old)
+                if line[0] in ' +':
+                    new.append(line[1:]); previous.append(new)
+            i += 1
+        require(len(old)==old_count and len(new)==new_count, 'invalid_unified_patch')
+        hunks.append(Hunk(old_no-1 if old_count else old_no,
+                          new_no-1 if new_count else new_no, tuple(old), tuple(new)))
+    require(bool(hunks), 'invalid_unified_patch')
+    return hunks
+
+
+def _unified(source, hunks, *, rebase=False):
+    lines = _diff_lines(source)
+    require(len(lines) <= MAX_LINES, 'patch_too_complex')
+    require(not rebase or len(lines)*len(hunks)<=1000000, 'patch_too_complex')
+    result = []
+    last = delta = 0
+    for hunk in hunks:
+        start = hunk.old_start
+        if rebase:
+            # Insertions without an unchanged context have no safe locator.
+            require(bool(hunk.old), 'patch_rebase_unsupported')
+            # Exact sequence search in linear time (KMP), not quadratic scanning
+            # of a large context at every repeated line.
+            starts = _sequence_matches(lines,hunk.old)
+            require(bool(starts), 'patch_no_match')
+            require(len(starts)==1, 'patch_ambiguous')
+            start = starts[0]
+        else:
+            require(hunk.new_start==hunk.old_start+delta, 'invalid_unified_patch')
+        require(last<=start<=len(lines), 'invalid_unified_patch')
+        end = start+len(hunk.old)
+        require(tuple(lines[start:end])==hunk.old, 'patch_no_match')
+        result.extend(lines[last:start]); result.extend(hunk.new)
+        last = end
+        delta += len(hunk.new)-len(hunk.old)
+    result.extend(lines[last:])
+    return ''.join(result)
+
+
+def _sequence_matches(lines, needle):
+    prefix = [0]*len(needle)
+    j = 0
+    for i in range(1,len(needle)):
+        while j and needle[i]!=needle[j]:
+            j = prefix[j-1]
+        if needle[i]==needle[j]:
+            j += 1
+        prefix[i] = j
+    result = []
+    j = 0
+    for i,line in enumerate(lines):
+        while j and line!=needle[j]:
+            j = prefix[j-1]
+        if line==needle[j]:
+            j += 1
+        if j==len(needle):
+            result.append(i-j+1)
+            if len(result)==2:
+                return result
+            j = prefix[j-1]
+    return result
+
+
+_ATX = re.compile(r'^ {0,3}(#{1,6})(?:[ \t]+(.*?)|[ \t]*)$')
+_FENCE = re.compile(r'^ {0,3}(`{3,}|~{3,})(.*)$')
+_SETEXT = re.compile(r'^ {0,3}(=+|-+)[ \t]*$')
+
+
+def _markdown(source):
+    """Locate raw spans; this is intentionally not a semantic Markdown AST."""
+    # Markdown admits CR/LF/CRLF, not Unicode paragraph/line separators.
+    lines = re.findall(r'[^\r\n]*(?:\r\n|\r|\n)|[^\r\n]+$', source)
+    require(len(lines)<=MAX_LINES, 'patch_too_complex')
+    offsets = [0]
+    for line in lines:
+        offsets.append(offsets[-1]+len(line))
+    headings, blocks = [], []
+    i = 0
+    while i<len(lines):
+        text = lines[i].rstrip('\r\n')
+        if not text.strip():
+            i += 1
+            continue
+        fence = _FENCE.match(text)
+        if fence and not (fence[1][0]=='`' and '`' in fence[2]):
+            start = i
+            marker = fence[1]
+            i += 1
+            while i<len(lines):
+                end = _FENCE.match(lines[i].rstrip('\r\n'))
+                i += 1
+                if end and end[1][0]==marker[0] and len(end[1])>=len(marker) and not end[2].strip():
+                    break
+            blocks.append((offsets[start],offsets[i]))
+            continue
+        atx = _ATX.match(text)
+        setext = _SETEXT.match(lines[i+1].rstrip('\r\n')) if i+1<len(lines) else None
+        # Four-space-indented code is not a setext heading.
+        if atx or (setext and not text.startswith(('    ','\t'))):
+            level = len(atx[1]) if atx else (1 if setext[1][0]=='=' else 2)
+            title = (atx[2] or '').strip() if atx else text.strip()
+            if atx:
+                title = re.sub(r'[ \t]+#+[ \t]*$','',title).rstrip()
+            start = i
+            i += 1 if atx else 2
+            headings.append((title,level,offsets[start]))
+            blocks.append((offsets[start],offsets[i]))
+            continue
+        start = i
+        i += 1
+        while i<len(lines) and lines[i].strip():
+            upcoming = lines[i].rstrip('\r\n')
+            underline = _SETEXT.match(lines[i+1].rstrip('\r\n')) if i+1<len(lines) else None
+            if _ATX.match(upcoming) or _FENCE.match(upcoming) or underline:
+                break
+            i += 1
+        blocks.append((offsets[start],offsets[i]))
+    require(len(blocks)<=PATCH_CANDIDATE_LIMIT, 'patch_too_complex')
+    return headings,blocks
+
+
+def _structured(source, patch):
+    headings, blocks = _markdown(source)
+    if patch['kind']=='heading':
+        matches = [i for i,(title,_,_) in enumerate(headings) if title==patch['heading']]
+        require(bool(matches), 'patch_no_match')
+        require(len(matches)==1, 'patch_ambiguous')
+        index = matches[0]
+        _, level, start = headings[index]
+        end = next((pos for _,depth,pos in headings[index+1:] if depth<=level),len(source))
+        matches = [(start,end)]
+    else:
+        matches = [(start,end) for start,end in blocks
+                   if digest(source[start:end].encode('utf-8'))==patch['digest']]
+        require(bool(matches), 'patch_no_match')
+        require(len(matches)==1, 'patch_ambiguous')
+    start,end = matches[0]
+    require(digest(source[start:end].encode('utf-8'))==patch['digest'], 'patch_no_match')
+    return source[:start]+patch['replacement']+source[end:]
+
+
+def apply_patch(source, patch, *, base_source=None):
+    validate_patch(patch)
+    require(len(source.encode('utf-8'))<=PATCH_LIMIT, 'patch_too_large')
+    if base_source is not None:
+        require(len(base_source.encode('utf-8'))<=PATCH_LIMIT, 'patch_too_large')
+        # Base validation is independent of current text: no forged old anchor
+        # can become a successful edit merely because it matches the new text.
+        apply_patch(base_source,patch)
+    kind = patch['kind']
+    if kind=='exact':
+        result = apply_text_patch(source,patch['exact'],patch['replacement'],
+                                  patch.get('before',''),patch.get('after',''))
+    elif kind=='unified':
+        result = _unified(source,_hunks(patch['diff']),rebase=base_source is not None)
+    else:
+        result = _structured(source,patch)
+    require(len(result.encode('utf-8'))<=PATCH_LIMIT, 'patch_too_large')
+    return result

msg/core/tool_execution.py

@@ -0,0 +1,28 @@
+"""Local tool execution port, independent of worker orchestration and backends.
+
+A runner returns a local artifact, never a committed ResourceRef. The worker
+validates it, rechecks current authority/attempt/deadline and publishes it in its
+existing transaction. Implementing this Protocol grants no execution authority.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from .models import JsonMap, NetworkPolicy, ToolSpec
+
+
+@dataclass(frozen=True, slots=True)
+class ToolResult:
+    path: Path
+    media_type: str
+    metadata: dict
+
+
+@runtime_checkable
+class ToolRunner(Protocol):
+    async def __call__(
+        self, tool: ToolSpec, arguments: JsonMap,
+        policies: tuple[NetworkPolicy, ...], directory: Path, /
+    ) -> ToolResult: ...

msg/core/transfer.py

@@ -0,0 +1,183 @@
+"""Persistent, owner-bound byte-range transfers with explicit publication points."""
+from __future__ import annotations
+
+from dataclasses import replace
+from datetime import timedelta
+import hashlib
+import re
+from uuid import uuid4
+
+from msg.core.codec import canonical,decode,digest,loads,wire
+from msg.core.errors import Failure,require
+from msg.core.models import TransferSession,TransferChunk,ResourceRef,AccessRequirement,Page
+from msg.storage.capacity import require_transfer_capacity
+
+
+def validate_digest(value):
+    require(isinstance(value,str) and re.fullmatch(r'sha256:[0-9a-f]{64}',value) is not None,'invalid_digest')
+    return value
+
+
+class TransferService:
+    def __init__(self,metadata,contents,authorizer,clock, *, publish,ttl=86400,part_bytes=65536):
+        self.metadata,self.contents,self.authorizer,self.clock=metadata,contents,authorizer,clock
+        self.publish,self.ttl,self.part_bytes=publish,ttl,part_bytes
+
+    async def _access(self,context,tx,resource,operation,check):
+        await self.authorizer.require(context,None,(AccessRequirement(resource_id=resource,
+            operation=operation+'@1',check=check),),tx)
+
+    async def _session(self,context,tx,id,operation, *, allow_cancelled=False):
+        transfer=await tx.transfer(id)
+        require(transfer.subject_id==context.principal.subject,'transfer_owner_required')
+        require(transfer.expires_at>self.clock(),'transfer_expired')
+        require(allow_cancelled or transfer.state not in {'cancelled','expired'},'transfer_closed')
+        # Each continuation rechecks the current permission and credential ceiling,
+        # not just a cached decision from open().
+        check='create' if transfer.direction=='upload' else 'read'
+        await self._access(context,tx,transfer.target.id,operation,check)
+        return transfer
+
+    def _limits(self,tx,id):
+        return loads(tx.one('SELECT limits FROM transfers WHERE id=?',(id,))[0])
+
+    async def open(self,context,direction,target,size,digest,expires_at, *, limits=None,media_type='application/octet-stream'):
+        require(context.principal.subject is not None,'authentication_required')
+        require(direction in {'upload','download'},'invalid_direction')
+        require(self.clock()<expires_at<=self.clock()+timedelta(seconds=self.ttl),'invalid_transfer_expiry')
+        require(size is None or type(size) is int and 0<=size<2**63,'invalid_size')
+        if digest is not None:
+            validate_digest(digest)
+        require(isinstance(media_type,str) and 0<len(media_type)<=255 and '\r' not in media_type and '\n' not in media_type,
+                'invalid_media_type')
+        async with self.metadata.transaction(write=True) as tx:
+            require_transfer_capacity(tx, new_transfer=True)
+            if direction=='download':
+                require(target is not None,'target_required')
+                await self._access(context,tx,target.id,'transfer.open','read')
+                revision=await tx.revision(target)
+                target=ResourceRef(id=target.id,revision=revision.id)
+                size,digest=revision.content.size,revision.content.digest
+                media_type=revision.content.media_type
+            else:
+                if target is None:
+                    row=tx.one('SELECT id FROM resources WHERE parent=? AND name=?',(context.principal.subject,'files'))
+                    require(row is not None,'upload_parent_required')
+                    target=ResourceRef(id=row[0])
+                require(target.revision is None,'upload_target_must_be_container')
+                parent=await tx.resource(target.id)
+                require(parent.type in {'topic','user','organization'},'not_a_container')
+                await self._access(context,tx,target.id,'transfer.open','create')
+            transfer=TransferSession(id='tr_'+uuid4().hex,subject_id=context.principal.subject,direction=direction,
+                state='open',target=target,expected_size=size,expected_digest=digest,expires_at=expires_at,generation=0)
+            await tx.save_transfer(transfer,None)
+            options=dict(limits or {},media_type=media_type)
+            options.setdefault('part_bytes',self.part_bytes)
+            tx.execute('UPDATE transfers SET limits=? WHERE id=?',(canonical(options).decode(),transfer.id),write=True)
+            return transfer
+
+    async def put(self,context,transfer_id,offset,data,digest):
+        validate_digest(digest)
+        require(type(offset) is int and 0<=offset<2**63,'invalid_offset')
+        require(isinstance(data,bytes) and data,'empty_chunk')
+        require('sha256:'+hashlib.sha256(data).hexdigest()==digest,'chunk_digest_mismatch')
+        async with self.metadata.transaction(write=True) as tx:
+            transfer=await self._session(context,tx,transfer_id,'transfer.part_put')
+            require(transfer.direction=='upload','wrong_transfer_direction')
+            limits=self._limits(tx,transfer_id)
+            require(len(data)<=limits['part_bytes'],'part_too_large')
+            require(offset+len(data)<2**63,'invalid_offset')
+            require(transfer.expected_size is None or offset+len(data)<=transfer.expected_size,'part_out_of_bounds')
+            # Equality includes the range and bytes' digest, not their arrival order.
+            rows=tx.rows('SELECT offset,length,body FROM chunks WHERE transfer_id=? AND offset<? AND offset+length>?',
+                         (transfer_id,offset+len(data),offset))
+            if rows:
+                require(len(rows)==1 and rows[0][0]==offset and rows[0][1]==len(data) and
+                    decode(TransferChunk,loads(rows[0][2])).content.digest==digest,'chunk_conflict')
+                return decode(TransferChunk,loads(rows[0][2]))
+            require(transfer.state=='open','transfer_closed')
+            require_transfer_capacity(tx, len(data))
+            async def pieces():
+                yield data
+            blob=await self.contents.put(pieces(),'application/octet-stream',expected_digest=digest)
+            lease=transfer_id+':'+str(offset)
+            tx.on_rollback(lambda: self.contents.unpin(blob,lease))
+            await self.contents.pin(blob,lease)
+            chunk=TransferChunk(transfer_id=transfer_id,offset=offset,content=blob)
+            await tx.put_chunk(chunk)
+            await tx.save_transfer(replace(transfer,generation=transfer.generation+1),transfer.generation)
+            return chunk
+
+    async def get(self,context,transfer_id,byte_range):
+        require(isinstance(byte_range,tuple) and len(byte_range)==2 and
+            all(type(i) is int for i in byte_range),'invalid_byte_range')
+        start,end=byte_range
+        async with self.metadata.transaction(write=False) as tx:
+            transfer=await self._session(context,tx,transfer_id,'transfer.part_get')
+            require(transfer.direction=='download','wrong_transfer_direction')
+            require(0<=start<=end<=transfer.expected_size,'part_out_of_bounds')
+            limits=self._limits(tx,transfer_id)
+            require(end-start<=limits['part_bytes'],'part_too_large')
+            revision=await tx.revision(transfer.target)
+            data=b''.join([part async for part in self.contents.read(revision.content,(start,end))])
+            from msg.core.models import BlobRef
+            chunk=TransferChunk(transfer_id=transfer_id,offset=start,
+                content=BlobRef(digest=digest(data),size=len(data),media_type=revision.content.media_type))
+            return chunk,data
+
+    async def status(self,context,transfer_id,cursor=None,limit=50):
+        async with self.metadata.transaction(write=False) as tx:
+            transfer=await self._session(context,tx,transfer_id,'transfer.status',allow_cancelled=True)
+            if transfer.direction=='download' or transfer.state=='sealed':
+                return Page(items=())
+            if transfer.expected_size is None:
+                # An unknown final length cannot be described as a closed missing
+                # interval. Expose received ranges through the operation projection.
+                return Page(items=())
+            return await tx.missing_ranges(transfer_id,cursor,limit)
+
+    async def seal(self,context,transfer_id,final_size,final_digest):
+        validate_digest(final_digest)
+        require(type(final_size) is int and 0<=final_size<2**63,'invalid_size')
+        async with self.metadata.transaction(write=True) as tx:
+            transfer=await self._session(context,tx,transfer_id,'transfer.seal')
+            require(transfer.expected_size is None or final_size==transfer.expected_size,'final_size_mismatch')
+            require(transfer.expected_digest is None or final_digest==transfer.expected_digest,'final_digest_mismatch')
+            if transfer.state=='sealed':
+                require(transfer.expected_size==final_size and transfer.expected_digest==final_digest,'seal_conflict')
+                return transfer.output
+            require(transfer.state=='open','transfer_closed')
+            if transfer.direction=='download':
+                output=transfer.target
+            else:
+                position=0
+                # Rows are streamed from SQLite, not collected into an unbounded list.
+                for start,length in tx.execute('SELECT offset,length FROM chunks WHERE transfer_id=? ORDER BY offset',(transfer_id,)):
+                    require(start==position,'transfer_incomplete')
+                    position=start+length
+                require(position==final_size,'transfer_incomplete')
+                async def pieces():
+                    for (body,) in tx.execute('SELECT body FROM chunks WHERE transfer_id=? ORDER BY offset',(transfer_id,)):
+                        chunk=decode(TransferChunk,loads(body))
+                        async for data in self.contents.read(chunk.content):
+                            yield data
+                limits=self._limits(tx,transfer_id)
+                blob=await self.contents.put(pieces(),limits['media_type'],expected_digest=final_digest)
+                require(blob.size==final_size,'final_size_mismatch')
+                tx.on_rollback(lambda: self.contents.unpin(blob,transfer_id+':sealed'))
+                await self.contents.pin(blob,transfer_id+':sealed')
+                output=await self.publish(context,tx,transfer.target.id,blob)
+            await tx.save_transfer(replace(transfer,state='sealed',expected_size=final_size,expected_digest=final_digest,
+                output=output,generation=transfer.generation+1),transfer.generation)
+            return output
+
+    async def cancel(self,context,transfer_id):
+        async with self.metadata.transaction(write=True) as tx:
+            transfer=await self._session(context,tx,transfer_id,'transfer.cancel',allow_cancelled=True)
+            if transfer.state=='cancelled':
+                return
+            require(transfer.state=='open','transfer_closed')
+            await tx.save_transfer(replace(transfer,state='cancelled',generation=transfer.generation+1),transfer.generation)
+            # Physical chunk cleanup is separate from this SQL transaction. Its
+            # failure cannot invalidate a cancellation that has already committed.
+            tx.set_setting('transfer_cleanup:'+transfer.id,{'state':'pending','after':wire(self.clock())})