synapse 2.219.0__py311-none-any.whl → 2.220.0__py311-none-any.whl

synapse/lib/stormlib/imap.py

@@ -1,22 +1,458 @@
+import random
 import asyncio
+import imaplib
+import logging
 
-import aioimaplib
+import lark
+import regex
 
 import synapse.exc as s_exc
+import synapse.data as s_data
 import synapse.common as s_common
+
 import synapse.lib.coro as s_coro
+import synapse.lib.link as s_link
 import synapse.lib.stormtypes as s_stormtypes
 
-async def run_imap_coro(coro):
+logger = logging.getLogger(__name__)
+
+CRLF = b'\r\n'
+CRLFLEN = len(CRLF)
+UNTAGGED = '*'
+
+TAGVAL_MIN = 4096
+TAGVAL_MAX = 65535
+
+def quote(text, escape=True):
+    if text == '""':
+        # Don't quote empty string
+        return text
+
+    if ' ' not in text and '"' not in text and '\\' not in text:
+        return text
+
+    text = text.replace('\\', '\\\\')
+    text = text.replace('"', '\\"')
+
+    return f'"{text}"'
+
+_grammar = s_data.getLark('imap')
+LarkParser = lark.Lark(_grammar, regex=True, start='input',
+                       maybe_placeholders=False, propagate_positions=True, parser='lalr')
+
+class AstConverter(lark.Transformer):
+    def quoted(self, args):
+        return ''.join(args)
+
+    def unquoted(self, args):
+        return ''.join(args)
+
+def qsplit(text):
+    '''
+    Split on spaces.
+    Preserve quoted strings.
+    Unescape backslash and double quotes.
+    Unquote quoted strings.
+
+    Raise BadDataValu if:
+        - quotes are unclosed.
+        - quoted strings don't have a space before/after (not including beginning/end of line).
+        - double-quotes or backslashes are escaped outside of a quoted string.
+    '''
+    def on_error(exc):
+        # Escaped double-quote or backslash not in quotes
+        if exc.token_history and len(exc.token_history) == 1 and (tok := exc.token_history[0]).type == 'UNQUOTED_CHAR' and tok.value == '\\':
+            mesg = f'Invalid data: {exc.token.value} cannot be escaped.'
+            raise s_exc.BadDataValu(mesg=mesg, data=text) from None
+
+        if exc.token.type == 'UNQUOTED_CHAR' and exc.expected == {'QUOTED_SPECIALS'}:
+            mesg = f'Invalid data: {exc.token.value} cannot be escaped.'
+            raise s_exc.BadDataValu(mesg=mesg, data=text) from None
+
+        # Double quote (opening a quoted string) at end of line
+        if exc.token.type == 'DBLQUOTE' and exc.column == len(text):
+            mesg = 'Quoted strings must be preceded and followed by a space.'
+            raise s_exc.BadDataValu(mesg=mesg, data=text) from None
+
+        # Unclosed quoted string
+        if exc.token.type == '$END' and exc.column == len(text) and exc.expected == {'QUOTED_CHAR', 'DBLQUOTE', 'BACKSLASH'}:
+            mesg = 'Unclosed quotes in text.'
+            raise s_exc.BadDataValu(mesg=mesg, data=text) from None
+
+        # Catch-all exception
+        raise s_exc.BadDataValu(mesg='Unable to parse IMAP response data.', data=text) from None # pragma: no cover
+
+    tree = LarkParser.parse(text, on_error=on_error)
+    newtree = AstConverter(text).transform(tree)
+    return newtree.children
+
+imap_rgx = regex.compile(
+    br'''
+    ^
+      (?P<tag>\*|\+|[0-9a-zA-Z]+)       # tag is mandatory
+      (\s(?P<uid>[0-9]+))?              # uid is optional
+      (\s(?P<response>[A-Z]{2,}))       # response is mandatory
+      (\s\[(?P<code>.*?)\])?            # code is optional
+      (\s(?P<data>.*?(?! {\d+})))?      # data is optional
+      (\s({(?P<size>\d+)}))?            # size is optional
+    $
+    ''',
+    flags=regex.VERBOSE
+)
+
+imap_rgx_cont = regex.compile(
+    br'''
+    ^
+      ((?P<data>.*?(?! {\d+})))?        # data is optional
+      (\s({(?P<size>\d+)}))?            # size is optional
+    $
+    ''',
+    flags=regex.VERBOSE
+)
+
+class IMAPBase(s_link.Link):
+    '''
+    Base class for IMAPClient and IMAPServer (in test_lib_stormlib_imap.py).
+    '''
+    async def __anit__(self, reader, writer, info=None, forceclose=False):
+        await s_link.Link.__anit__(self, reader, writer, info=info, forceclose=forceclose)
+
+        self._rxbuf = b''
+        self.state = 'LOGOUT'
+
+    def _parseLine(self, line): # pragma: no cover
+        raise NotImplementedError('Not implemented')
+
+    def pack(self, mesg): # pragma: no cover
+        raise NotImplementedError('Not implemented')
+
+    def feed(self, byts):
+        ret = []
+
+        # Append new bytes to existing bytes
+        self._rxbuf += byts
+
+        # Iterate through buffer and parse out (up to 32) complete messages.
+        # NB: The 32 message maximum is an arbitrary number to keep this loop
+        # from running forever with an endless number of messages from the
+        # server.
+        while (offs := self._rxbuf.find(CRLF)) != -1 and len(ret) < 32:
+
+            # Get the line out of the buffer
+            line = self._rxbuf[:offs]
+
+            # Parse line
+            mesg = self._parseLine(line)
+
+            end = offs + CRLFLEN
+
+            # Handle continuations
+            while (size := mesg.get('size')) is not None:
+                start = end
+                end = start + size
+
+                # Check for complete data
+                if len(self._rxbuf) < start + end - start: # pragma: no cover
+                    return ret
+
+                # Check for end of message
+                if (offs := self._rxbuf[end:].find(CRLF)) == -1: # pragma: no cover
+                    return ret
+
+                # Extract the attachment and add it to the message
+                attachment = self._rxbuf[start:end]
+                mesg['attachments'].append(attachment)
+
+                msgdata = self._rxbuf[end:end + offs]
+
+                # Get the data and/or size from the trailing message data
+                cont = imap_rgx_cont.match(msgdata).groupdict()
+                if (size := cont.get('size')) is not None:
+                    size = int(size)
+
+                mesg['size'] = size
+
+                contdata = cont.get('data', b'')
+                mesg['data'] += contdata
+
+                end = end + offs + CRLFLEN
+
+            # Increment buffer
+            self._rxbuf = self._rxbuf[end:]
+
+            # Log only under __debug__ because there might be sensitive info like passwords
+            if __debug__:
+                logger.debug('%s RECV: %s', self.__class__.__name__, mesg)
+
+            ret.append((None, mesg))
+
+        return ret
+
+class IMAPClient(IMAPBase):
+    async def postAnit(self):
+        self._tagval = random.randint(TAGVAL_MIN, TAGVAL_MAX)
+        self.readonly = False
+        self.capabilities = []
+
+        # Get and handle the server greeting
+        response = await self.getResponse()
+        greeting = response.get(UNTAGGED)[0]
+
+        if greeting.get('response') == 'PREAUTH':
+            self.state = 'AUTH'
+        elif greeting.get('response') == 'OK':
+            self.state = 'NONAUTH'
+        else:
+            # Includes greeting.get('response') == 'BYE'
+            raise s_exc.ImapError(mesg=greeting.get('data').decode(), response=response)
+
+        # Some servers will list capabilities in the greeting
+        if (code := greeting.get('code')) is not None and code.startswith('CAPABILITY'):
+            self.capabilities = qsplit(code)[1:]
+
+        if not self.capabilities:
+            (ok, data) = await self.capability()
+            if not ok:
+                mesg = data[0].decode()
+                raise s_exc.ImapError(mesg=mesg)
+
+        return self
+
+    def _parseLine(self, line):
+        match = imap_rgx.match(line)
+        if match is None:
+            mesg = 'Unable to parse response from server.'
+            raise s_exc.ImapError(mesg=mesg, data=line)
+
+        mesg = match.groupdict()
+
+        for key, valu in mesg.items():
+            if key == 'data' or valu is None:
+                continue
+
+            mesg[key] = valu.decode()
+
+        if mesg.get('data') is None:
+            mesg['data'] = b''
+
+        if (uid := mesg.get('uid')) is not None:
+            mesg['uid'] = int(uid)
+
+        if (size := mesg.get('size')) is not None:
+            mesg['size'] = int(size)
+
+        # For attaching continuation data
+        mesg['attachments'] = []
+
+        return mesg
+
+    async def pack(self, mesg):
+        (tag, command, args) = mesg
+
+        cmdargs = ''
+        if args:
+            cmdargs = ' ' + ' '.join(args)
+
+        mesg = f'{tag} {command}{cmdargs}\r\n'
+
+        # Log only under __debug__ because there might be sensitive info like passwords
+        if __debug__:
+            logger.debug('%s SEND: %s', self.__class__.__name__, mesg)
+
+        return mesg.encode()
+
+    async def getResponse(self, tag=None):
+        resp = {}
+
+        while True:
+            msg = await self.rx()
+
+            mtag = msg.get('tag')
+            resp.setdefault(mtag, []).append(msg)
+
+            if tag is None or mtag == tag:
+                break
+
+        return resp
+
+    def _genTag(self):
+        self._tagval = (self._tagval + 1) % TAGVAL_MAX
+        if self._tagval == 0:
+            self._tagval = TAGVAL_MIN
+
+        return imaplib.Int2AP(self._tagval).decode()
+
+    async def _command(self, tag, command, *args):
+        if command.upper() not in imaplib.Commands:
+            mesg = f'Unsupported command: {command}.'
+            raise s_exc.ImapError(mesg=mesg, command=command)
+
+        if self.state not in imaplib.Commands.get(command.upper()):
+            mesg = f'{command} not allowed in the {self.state} state.'
+            raise s_exc.ImapError(mesg=mesg, state=self.state, command=command)
+
+        await self.tx((tag, command, args))
+        return await self.getResponse(tag)
+
+    def okSetState(self, response, state):
+        if response.get('response') == 'OK':
+            self.state = state
+
+    async def capability(self):
+        tag = self._genTag()
+        resp = await self._command(tag, 'CAPABILITY')
+
+        response = resp.get(tag)[0]
+        if response.get('response') != 'OK':
+            return False, [response.get('data')]
+
+        if len(untagged := resp.get(UNTAGGED, [])) != 1:
+            return False, [b'Invalid server response.']
+
+        capabilities = untagged[0].get('data').decode()
+        self.capabilities = qsplit(capabilities)
+
+        return True, [capabilities]
+
+    async def login(self, user, passwd):
+        if 'AUTH=PLAIN' not in self.capabilities:
+            return False, [b'Plain authentication not available on server.']
+
+        if 'LOGINDISABLED' in self.capabilities:
+            return False, [b'Login disabled on server.']
+
+        tag = self._genTag()
+        resp = await self._command(tag, 'LOGIN', quote(user), quote(passwd))
+
+        response = resp.get(tag)[0]
+        if response.get('response') != 'OK':
+            return False, [response.get('data')]
+
+        # Some servers will update capabilities with the login response
+        if (code := response.get('code')) is not None and code.startswith('CAPABILITY'):
+            self.capabilities = qsplit(code)[1:]
+
+        self.okSetState(response, 'AUTH')
+
+        return True, [response.get('data')]
+
+    async def select(self, mailbox='INBOX'):
+        tag = self._genTag()
+        resp = await self._command(tag, 'SELECT', quote(mailbox))
+
+        response = resp.get(tag)[0]
+        if response.get('response') != 'OK':
+            return False, [response.get('data')]
+
+        if (code := response.get('code')) is not None:
+            if 'READ-ONLY' in code:
+                self.readonly = True
+
+            if 'READ-WRITE' in code:
+                self.readonly = False
+
+        self.okSetState(response, 'SELECTED')
+        return True, [response.get('data')]
+
+    async def list(self, refname, pattern):
+        tag = self._genTag()
+        resp = await self._command(tag, 'LIST', quote(refname), quote(pattern))
+
+        response = resp.get(tag)[0]
+        if response.get('response') != 'OK':
+            return False, [response.get('data')]
+
+        data = []
+        for mesg in resp.get(UNTAGGED, []):
+            data.append(mesg.get('data'))
+
+        return True, data
+
+    async def uid_store(self, uidset, dataname, datavalu):
+        if self.readonly:
+            return False, [b'Selected mailbox is read-only.']
+
+        args = f'{uidset} {dataname} {datavalu}'
+        return await self.uid('STORE', args)
+
+    async def uid_search(self, *criteria, charset='UTF-8'):
+        args = ''
+        if charset is not None:
+            args += f'CHARSET {charset} '
+        args += ' '.join(quote(c) for c in criteria)
+        return await self.uid('SEARCH', args)
+
+    async def uid_fetch(self, uidset, datanames):
+        args = f'{uidset} {datanames}'
+        return await self.uid('FETCH', args)
+
+    async def uid(self, cmdname, cmdargs):
+        tag = self._genTag()
+
+        resp = await self._command(tag, 'UID', cmdname, cmdargs)
+
+        response = resp.get(tag)[0]
+        if response.get('response') != 'OK':
+            return False, [response.get('data')]
+
+        untagged = resp.get(UNTAGGED, [])
+
+        if cmdname == 'FETCH':
+            # FETCH returns a list of attachments from each message followed by
+            # the message data. For example, a FETCH 4 (RFC822 BODY[HEADER])
+            # would return:
+            # [ <RFC822 message>, <BODY[HEADER] message>, '(UID 4 RFC822 BODY[HEADER])' ]
+            #
+            # This allows the consumer to get each of the requested data
+            # messages and then parse the message data to figure out which
+            # attachment is which.
+
+            ret = []
+            for u in untagged:
+                ret.extend(u.get('attachments'))
+                ret.append(u.get('data'))
+            return True, ret
+
+        return True, [u.get('data') for u in untagged]
+
+    async def expunge(self):
+        if self.readonly:
+            return False, [b'Selected mailbox is read-only.']
+
+        tag = self._genTag()
+        resp = await self._command(tag, 'EXPUNGE')
+
+        response = resp.get(tag)[0]
+        if response.get('response') != 'OK':
+            return False, [response.get('data')]
+
+        return True, [response.get('data')]
+
+    async def logout(self):
+        tag = self._genTag()
+        resp = await self._command(tag, 'LOGOUT')
+
+        response = resp.get(tag)[0]
+        if response.get('response') != 'OK':
+            return False, [response.get('data')]
+
+        untagged = resp.get(UNTAGGED, [])
+        if len(untagged) != 1 or untagged[0].get('response') != 'BYE':
+            return False, [b'Server failed to send expected BYE response.']
+
+        self.okSetState(response, 'LOGOUT')
+        return True, [response.get('data')]
+
+async def run_imap_coro(coro, timeout):
     '''
     Raises or returns data.
     '''
     try:
-        status, data = await coro
+        status, data = await s_common.wait_for(coro, timeout)
     except asyncio.TimeoutError:
-        raise s_exc.StormRuntimeError(mesg='Timed out waiting for IMAP server response.') from None
+        raise s_exc.TimeOut(mesg='Timed out waiting for IMAP server response.') from None
 
-    if status == 'OK':
+    if status:
         return data
 
     try:
@@ -24,7 +460,7 @@ async def run_imap_coro(coro):
     except (TypeError, AttributeError, IndexError, UnicodeDecodeError):
         mesg = 'IMAP server returned an error'
 
-    raise s_exc.StormRuntimeError(mesg=mesg, status=status)
+    raise s_exc.ImapError(mesg=mesg, status=status)
 
 @s_stormtypes.registry.registerLib
 class ImapLib(s_stormtypes.Lib):
@@ -72,7 +508,7 @@ class ImapLib(s_stormtypes.Lib):
             'connect': self.connect,
         }
 
-    async def connect(self, host, port=993, timeout=30, ssl=True, ssl_verify=True):
+    async def connect(self, host, port=imaplib.IMAP4_SSL_PORT, timeout=30, ssl=True, ssl_verify=True):
 
         self.runt.confirm(('storm', 'inet', 'imap', 'connect'))
 
@@ -82,24 +518,26 @@ class ImapLib(s_stormtypes.Lib):
         ssl_verify = await s_stormtypes.tobool(ssl_verify)
         timeout = await s_stormtypes.toint(timeout, noneok=True)
 
+        ctx = None
         if ssl:
             ctx = self.runt.snap.core.getCachedSslCtx(opts=None, verify=ssl_verify)
-            imap_cli = aioimaplib.IMAP4_SSL(host=host, port=port, timeout=timeout, ssl_context=ctx)
-        else:
-            imap_cli = aioimaplib.IMAP4(host=host, port=port, timeout=timeout)
-
-        async def fini():
-            # call protocol.logout() via a background task
-            s_coro.create_task(s_common.wait_for(imap_cli.protocol.logout(), 5))
 
-        self.runt.snap.onfini(fini)
+        coro = s_link.connect(host=host, port=port, ssl=ctx, linkcls=IMAPClient)
 
         try:
-            await imap_cli.wait_hello_from_server()
+            imap = await s_common.wait_for(coro, timeout)
         except asyncio.TimeoutError:
-            raise s_exc.StormRuntimeError(mesg='Timed out waiting for IMAP server hello.') from None
+            raise s_exc.TimeOut(mesg='Timed out waiting for IMAP server hello.') from None
 
-        return ImapServer(self.runt, imap_cli)
+        async def fini():
+            async def _logout():
+                await s_common.wait_for(imap.logout(), 5)
+                await imap.fini()
+            s_coro.create_task(_logout())
+
+        self.runt.snap.onfini(fini)
+
+        return ImapServer(self.runt, imap, timeout)
 
 @s_stormtypes.registry.registerType
 class ImapServer(s_stormtypes.StormType):
@@ -277,10 +715,11 @@ class ImapServer(s_stormtypes.StormType):
     )
     _storm_typename = 'inet:imap:server'
 
-    def __init__(self, runt, imap_cli, path=None):
+    def __init__(self, runt, imap_cli, timeout, path=None):
         s_stormtypes.StormType.__init__(self, path=path)
         self.runt = runt
         self.imap_cli = imap_cli
+        self.timeout = timeout
         self.locls.update(self.getObjLocals())
 
     def getObjLocals(self):
@@ -299,7 +738,7 @@ class ImapServer(s_stormtypes.StormType):
         passwd = await s_stormtypes.tostr(passwd)
 
         coro = self.imap_cli.login(user, passwd)
-        await run_imap_coro(coro)
+        await run_imap_coro(coro, self.timeout)
 
         return True, None
 
@@ -308,13 +747,11 @@ class ImapServer(s_stormtypes.StormType):
         reference_name = await s_stormtypes.tostr(reference_name)
 
         coro = self.imap_cli.list(reference_name, pattern)
-        data = await run_imap_coro(coro)
+        data = await run_imap_coro(coro, self.timeout)
 
         names = []
         for item in data:
-            if item == b'Success':
-                break
-            names.append(item.split(b' ')[-1].decode().strip('"'))
+            names.append(qsplit(item.decode())[-1])
 
         return True, names
 
@@ -322,17 +759,18 @@ class ImapServer(s_stormtypes.StormType):
         mailbox = await s_stormtypes.tostr(mailbox)
 
         coro = self.imap_cli.select(mailbox=mailbox)
-        await run_imap_coro(coro)
+        await run_imap_coro(coro, self.timeout)
 
         return True, None
 
     async def search(self, *args, charset='utf-8'):
         args = [await s_stormtypes.tostr(arg) for arg in args]
         charset = await s_stormtypes.tostr(charset, noneok=True)
+
         coro = self.imap_cli.uid_search(*args, charset=charset)
-        data = await run_imap_coro(coro)
+        data = await run_imap_coro(coro, self.timeout)
 
-        uids = data[0].decode().split(' ') if data[0] else []
+        uids = qsplit(data[0].decode()) if data[0] else []
         return True, uids
 
     async def fetch(self, uid):
@@ -344,10 +782,13 @@ class ImapServer(s_stormtypes.StormType):
         await self.runt.snap.core.getAxon()
         axon = self.runt.snap.core.axon
 
-        coro = self.imap_cli.uid('FETCH', str(uid), '(RFC822)')
-        data = await run_imap_coro(coro)
+        coro = self.imap_cli.uid_fetch(str(uid), '(RFC822)')
+        data = await run_imap_coro(coro, self.timeout)
+
+        if not data:
+            return False, f'No data received from fetch request for uid {uid}.'
 
-        size, sha256b = await axon.put(data[1])
+        size, sha256b = await axon.put(data[0])
 
         props = await axon.hashset(sha256b)
         props['size'] = size
@@ -359,18 +800,18 @@ class ImapServer(s_stormtypes.StormType):
     async def delete(self, uid_set):
         uid_set = await s_stormtypes.tostr(uid_set)
 
-        coro = self.imap_cli.uid('STORE', uid_set, '+FLAGS.SILENT (\\Deleted)')
-        await run_imap_coro(coro)
+        coro = self.imap_cli.uid_store(uid_set, '+FLAGS.SILENT', '(\\Deleted)')
+        await run_imap_coro(coro, self.timeout)
 
         coro = self.imap_cli.expunge()
-        await run_imap_coro(coro)
+        await run_imap_coro(coro, self.timeout)
 
         return True, None
 
     async def markSeen(self, uid_set):
         uid_set = await s_stormtypes.tostr(uid_set)
 
-        coro = self.imap_cli.uid('STORE', uid_set, '+FLAGS.SILENT (\\Seen)')
-        await run_imap_coro(coro)
+        coro = self.imap_cli.uid_store(uid_set, '+FLAGS.SILENT', '(\\Seen)')
+        await run_imap_coro(coro, self.timeout)
 
         return True, None

synapse/lib/stormtypes.py

@@ -4967,6 +4967,24 @@ class Bytes(Prim):
                       {'name': 'offset', 'type': 'int', 'desc': 'An offset to begin unpacking from.', 'default': 0},
                   ),
                   'returns': {'type': 'list', 'desc': 'The unpacked primitive values.', }}},
+        {'name': 'xor', 'desc': '''
+            Perform an "exclusive or" bitwise operation on the bytes and another set of bytes.
+
+            Notes:
+                The key bytes provided as an argument will be repeated as needed until all bytes have been
+                xor'd.
+
+                If a string is provided as the key argument, it will be utf8 encoded before being xor'd.
+
+            Examples:
+                Perform an xor operation on the bytes in $encoded using the bytes in $key::
+
+                    $decoded = $encoded.xor($key)''',
+         'type': {'type': 'function', '_funcname': '_methXor',
+                  'args': (
+                      {'name': 'key', 'type': ['str', 'bytes'], 'desc': 'The key bytes to perform the xor operation with.'},
+                  ),
+                  'returns': {'type': 'bytes', 'desc': "The xor'd bytes."}}},
     )
     _storm_typename = 'bytes'
     _ismutable = False
@@ -4977,6 +4995,7 @@ class Bytes(Prim):
 
     def getObjLocals(self):
         return {
+            'xor': self._methXor,
             'decode': self._methDecode,
             'bunzip': self._methBunzip,
             'gunzip': self._methGunzip,
@@ -5065,6 +5084,26 @@ class Bytes(Prim):
         except UnicodeDecodeError as e:
             raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(valu))}') from None
 
+    @stormfunc(readonly=True)
+    async def _methXor(self, key):
+        key = await toprim(key)
+        if isinstance(key, str):
+            key = key.encode()
+
+        if not isinstance(key, bytes):
+            raise s_exc.BadArg(mesg='$bytes.xor() key argument must be bytes or a str.')
+
+        if len(key) == 0:
+            raise s_exc.BadArg(mesg='$bytes.xor() key length must be greater than 0.')
+
+        arry = bytearray(self.valu)
+        keylen = len(key)
+
+        for i in range(len(arry)):
+            arry[i] ^= key[i % keylen]
+
+        return bytes(arry)
+
 @registry.registerType
 class Dict(Prim):
     '''

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 219, 0)
+version = (2, 220, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '9e442b257a7ddfa8d476eafdcad21c3a0440cc83'
+commit = '913156d6c195af916a32860341f285954cb4e882'