synapse 2.219.0__py311-none-any.whl → 2.220.0__py311-none-any.whl

synapse/data/__init__.py

@@ -29,5 +29,9 @@ def getJSON(name):
     with s_datfile.openDatFile(f'synapse.data/{name}.json') as fd:
         return s_json.load(fd)
 
+def getLark(name):
+    with s_datfile.openDatFile(f'synapse.data.lark/{name}.lark') as larkf:
+        return larkf.read().decode()
+
 def path(*names):
     return s_common.genpath(dirname, *names)

synapse/data/lark/imap.lark

@@ -0,0 +1,8 @@
+UNQUOTED_CHAR: /(?!\r|\n|\\"|\\\\)./
+QUOTED_SPECIALS: "\"" | "\\"
+
+QUOTED_CHAR: /(?!\r|\n|\\|\")./
+quoted: "\"" (QUOTED_CHAR | "\\" QUOTED_SPECIALS)* "\""
+unquoted: UNQUOTED_CHAR+
+_string: quoted | unquoted
+input: (_string " ")* _string

synapse/exc.py

@@ -216,6 +216,8 @@ class DupEdgeType(SynErr):
 
 class FileExists(SynErr): pass
 
+class ImapError(SynErr): pass
+
 class InconsistentStorage(SynErr):
     '''
     Stored persistent data is inconsistent

synapse/lib/json.py

@@ -88,12 +88,13 @@ def _dumps(obj, sort_keys=False, indent=False, default=None, newline=False):
         mesg = 'Object of type bytes is not JSON serializable'
         raise s_exc.MustBeJsonSafe(mesg=mesg)
 
-    # Raw strings have to be double-quoted. This is because the default behavior for `yyjson.Document`
-    #  is to attempt to parse the string as a serialized JSON string into objects, so we escape string
-    # values so we can get the JSON encoded string as output.
     if isinstance(obj, str) and obj not in ('null', 'true', 'false'):
-        # TODO in 3xx convert this into obj = f'''"{obj.replace('"', '\\"')}"'''
-        obj = ''.join(('"', obj.replace('"', '\\"'), '"'))
+        # Raw strings have to be double-quoted. This is because the default behavior for `yyjson.Document` is to attempt
+        # to parse the string as a serialized JSON string into objects. Instead of trying to manually escape the string,
+        # we wrap it in a list, serialize it, and then strip off the leading/trailing [] so we can get the JSON encoded
+        # string as output.
+        doc = yyjson.Document([obj], default=default, flags=rflags)
+        return doc.dumps(flags=wflags)[1:-1].encode()
 
     doc = yyjson.Document(obj, default=default, flags=rflags)
     return doc.dumps(flags=wflags).encode()

synapse/lib/link.py

@@ -18,34 +18,6 @@ import synapse.lib.msgpack as s_msgpack
 READSIZE = 16 * s_const.mebibyte
 MAXWRITE = 64 * s_const.mebibyte
 
-async def connect(host, port, ssl=None, hostname=None, linkinfo=None):
-    '''
-    Async connect and return a Link().
-    '''
-    info = {'host': host, 'port': port, 'ssl': ssl, 'hostname': hostname, 'tls': bool(ssl)}
-    if linkinfo is not None:
-        info.update(linkinfo)
-
-    ssl = info.get('ssl')
-    hostname = info.get('hostname')
-
-    reader, writer = await asyncio.open_connection(host, port, ssl=ssl, server_hostname=hostname)
-    return await Link.anit(reader, writer, info=info)
-
-async def listen(host, port, onlink, ssl=None):
-    '''
-    Listen on the given host/port and fire onlink(Link).
-
-    Returns a server object that contains the listening sockets
-    '''
-    async def onconn(reader, writer):
-        info = {'tls': bool(ssl)}
-        link = await Link.anit(reader, writer, info=info)
-        link.schedCoro(onlink(link))
-
-    server = await asyncio.start_server(onconn, host=host, port=port, ssl=ssl)
-    return server
-
 async def unixlisten(path, onlink):
     '''
     Start an PF_UNIX server listening on the given path.
@@ -247,30 +219,9 @@ class Link(s_base.Base):
         offs = 0
         size = len(byts)
 
-        async with self._txlock:
-
-            while offs < size:
-
-                self.writer.write(byts[offs:offs + MAXWRITE])
-                offs += MAXWRITE
-
-                await self.writer.drain()
-
-    async def tx(self, mesg):
-        '''
-        Async transmit routine which will wait for writer drain().
-        '''
-        if self.isfini:
-            raise s_exc.IsFini()
-
-        offs = 0
-        byts = s_msgpack.en(mesg)
-        size = len(byts)
-
         async with self._txlock:
 
             try:
-
                 while offs < size:
 
                     self.writer.write(byts[offs:offs + MAXWRITE])
@@ -287,6 +238,16 @@ class Link(s_base.Base):
 
                 raise
 
+    async def tx(self, mesg):
+        '''
+        Async transmit routine which will wait for writer drain().
+        '''
+        if self.isfini:
+            raise s_exc.IsFini()
+
+        byts = await self.pack(mesg)
+        await self.send(byts)
+
     def txfini(self):
         self.sock.shutdown(1)
 
@@ -353,6 +314,44 @@ class Link(s_base.Base):
 
     def feed(self, byts):
         '''
-        Used by Plex() to unpack bytes.
+        Used by rx() to unpack messages from bytes.
         '''
         return self.unpk.feed(byts)
+
+    async def pack(self, mesg):
+        '''
+        Used by tx() to pack messages into bytes
+        '''
+        return s_msgpack.en(mesg)
+
+async def connect(host, port, ssl=None, hostname=None, linkinfo=None, linkcls=Link):
+    '''
+    Async connect and return a <linkcls>.
+    '''
+    assert issubclass(linkcls, Link)
+
+    info = {'host': host, 'port': port, 'ssl': ssl, 'hostname': hostname, 'tls': bool(ssl)}
+    if linkinfo is not None:
+        info.update(linkinfo)
+
+    ssl = info.get('ssl')
+    hostname = info.get('hostname')
+
+    reader, writer = await asyncio.open_connection(host, port, ssl=ssl, server_hostname=hostname)
+    return await linkcls.anit(reader, writer, info=info)
+
+async def listen(host, port, onlink, ssl=None, linkcls=Link):
+    '''
+    Listen on the given host/port and fire onlink(<linkcls>).
+
+    Returns a server object that contains the listening sockets
+    '''
+    assert issubclass(linkcls, Link)
+
+    async def onconn(reader, writer):
+        info = {'tls': bool(ssl)}
+        link = await linkcls.anit(reader, writer, info=info)
+        link.schedCoro(onlink(link))
+
+    server = await asyncio.start_server(onconn, host=host, port=port, ssl=ssl)
+    return server

synapse/lib/parser.py

@@ -6,16 +6,16 @@ import lark  # type: ignore
 import regex  # type: ignore
 
 import synapse.exc as s_exc
+import synapse.data as s_data
 import synapse.common as s_common
 
 import synapse.lib.ast as s_ast
 import synapse.lib.coro as s_coro
 import synapse.lib.cache as s_cache
-import synapse.lib.datfile as s_datfile
 
 # TL;DR:  *rules* are the internal nodes of an abstract syntax tree (AST), *terminals* are the leaves
 
-# Note: this file is coupled strongly to synapse/lib/storm.lark.  Any changes to that file will probably require
+# Note: this file is coupled strongly to synapse/data/lark/storm.lark.  Any changes to that file will probably require
 # changes here
 
 # For easier-to-understand syntax errors
@@ -475,9 +475,7 @@ class AstConverter(lark.Transformer):
         kids[0].reverseLift(astinfo)
         return kids[0]
 
-with s_datfile.openDatFile('synapse.lib/storm.lark') as larkf:
-    _grammar = larkf.read().decode()
-
+_grammar = s_data.getLark('storm')
 LarkParser = lark.Lark(_grammar, regex=True, start=['query', 'lookup', 'cmdrargs', 'evalvalu', 'search'],
                        maybe_placeholders=False, propagate_positions=True, parser='lalr')
 

synapse/lib/storm.py

@@ -5796,6 +5796,7 @@ class IntersectCmd(Cmd):
         it:mitre:attack:group*in=(G0006, G0007) | intersect { -> it:mitre:attack:technique }
     '''
     name = 'intersect'
+    readonly = True
 
     def getArgParser(self):
         pars = Cmd.getArgParser(self)