synapse 2.176.0__py311-none-any.whl → 2.178.0__py311-none-any.whl

synapse/tests/test_lib_stormlib_gen.py

@@ -1,4 +1,5 @@
 import synapse.exc as s_exc
+import synapse.common as s_common
 
 import synapse.tests.utils as s_test
 
@@ -245,3 +246,101 @@ class StormLibGenTest(s_test.SynTest):
             names = nodes[0].get('names')
             self.len(1, names)
             self.isin('rhodesia', names)
+
+    async def test_stormlib_gen_fileBytes(self):
+
+        async with self.getTestCore() as core:
+            sha256 = s_common.buid().hex()
+            opts = {'vars': {'sha256': sha256}}
+
+            nodes = await core.nodes('yield $lib.gen.fileBytesBySha256($sha256)', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].get('sha256'), sha256)
+
+            sha256 = s_common.buid().hex()
+            opts = {'vars': {'sha256': sha256}}
+
+            q = '''
+                [ file:bytes=(file1,) :sha256=$sha256 ]
+                spin |
+                yield $lib.gen.fileBytesBySha256($sha256)
+            '''
+            nodes = await core.nodes(q, opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].repr(), 'guid:' + s_common.guid(('file1',)))
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('$lib.gen.fileBytesBySha256(newp)', opts=opts)
+
+            q = 'return($lib.gen.fileBytesBySha256(newp, try=$lib.true))'
+            self.none(await core.callStorm(q, opts=opts))
+
+    async def test_stormlib_gen_inetTlsServerCert(self):
+
+        async with self.getTestCore() as core:
+            sha256 = s_common.buid().hex()
+            opts = {'vars': {'sha256': sha256}}
+
+            q = '''
+                $server = {[ inet:server="1.2.3.4:443" ]}
+                yield $lib.gen.inetTlsServerCertByServerAndSha256($server, $sha256)
+            '''
+            nodes = await core.nodes(q, opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].get('server'), 'tcp://1.2.3.4:443')
+            cert = nodes[0].get('cert')
+            self.nn(cert)
+
+            nodes = await core.nodes('crypto:x509:cert:sha256=$sha256', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].repr(), cert)
+
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('$lib.gen.inetTlsServerCertByServerAndSha256(newp, $sha256)', opts=opts)
+
+            q = 'return($lib.gen.inetTlsServerCertByServerAndSha256(newp, $sha256, try=$lib.true))'
+            self.none(await core.callStorm(q, opts=opts))
+
+    async def test_stormlib_gen_cryptoX509Cert(self):
+
+        async with self.getTestCore() as core:
+
+            # Check guid generation
+            sha256 = s_common.buid().hex()
+            opts = {'vars': {'sha256': sha256}}
+            nodes = await core.nodes('yield $lib.gen.cryptoX509CertBySha256($sha256)', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].get('sha256'), sha256)
+            self.eq(nodes[0].repr(), s_common.guid(sha256))
+
+            # Check invalid values, no try
+            with self.raises(s_exc.BadTypeValu):
+                await core.callStorm('$lib.gen.cryptoX509CertBySha256(newp)')
+
+            # Check invalid values, with try
+            self.none(await core.callStorm('return($lib.gen.cryptoX509CertBySha256(newp, try=$lib.true))'))
+
+            # Check node matching with same sha256 values
+            sha256 = s_common.buid().hex()
+            opts = {'vars': {'sha256': sha256}}
+            nodes = await core.nodes('[crypto:x509:cert=* :sha256=$sha256]', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].get('sha256'), sha256)
+            self.ne(nodes[0].repr(), s_common.guid(sha256))
+            crypto = nodes[0].repr()
+
+            nodes = await core.nodes('yield $lib.gen.cryptoX509CertBySha256($sha256)', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].repr(), crypto)
+
+            # Check node matching, crypto:x509:cert -> file with matching sha256
+            sha256 = s_common.buid().hex()
+            opts = {'vars': {'sha256': sha256}}
+            nodes = await core.nodes('[crypto:x509:cert=* :file={[ file:bytes=$sha256 ]} ]', opts=opts)
+            self.len(1, nodes)
+            self.none(nodes[0].get('sha256'))
+            crypto = nodes[0].repr()
+
+            nodes = await core.nodes('yield $lib.gen.cryptoX509CertBySha256($sha256)', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].repr(), crypto)

synapse/tests/test_lib_stormlib_imap.py

@@ -1,3 +1,4 @@
+import ssl
 import asyncio
 
 from unittest import mock
@@ -124,8 +125,13 @@ class ImapTest(s_test.SynTest):
 
     async def test_storm_imap(self):
 
-        with mock.patch('aioimaplib.IMAP4.create_client', mock_create_client), \
-            mock.patch('aioimaplib.IMAP4_SSL.create_client', mock_create_client):
+        client_args = []
+        def client_mock(*args, **kwargs):
+            client_args.append((args, kwargs))
+            return mock_create_client(*args, **kwargs)
+
+        with mock.patch('aioimaplib.IMAP4.create_client', client_mock), \
+            mock.patch('aioimaplib.IMAP4_SSL.create_client', client_mock):
 
             async with self.getTestCore() as core:
 
@@ -137,16 +143,20 @@ class ImapTest(s_test.SynTest):
                 '''
                 retn = await core.callStorm(scmd)
                 self.eq((True, ('INBOX',)), retn)
+                ctx = self.nn(client_args[-1][0][5])  # type: ssl.SSLContext
+                self.eq(ctx.verify_mode, ssl.CERT_REQUIRED)
 
                 # search for UIDs
                 scmd = '''
-                    $server = $lib.inet.imap.connect(hello)
+                    $server = $lib.inet.imap.connect(hello, ssl_verify=(false))
                     $server.login("vtx@email.com", "secret")
                     $server.select("INBOX")
                     return($server.search("FROM", "foo@mail.com"))
                 '''
                 retn = await core.callStorm(scmd)
                 self.eq((True, ('8181', '8192', '8194')), retn)
+                ctx = self.nn(client_args[-1][0][5])  # type: ssl.SSLContext
+                self.eq(ctx.verify_mode, ssl.CERT_NONE)
 
                 # search for UIDs with specific charset
                 scmd = '''
@@ -186,6 +196,7 @@ class ImapTest(s_test.SynTest):
                 '''
                 retn = await core.callStorm(scmd)
                 self.eq((True, None), retn)
+                self.none(client_args[-1][0][5])
 
                 # delete
                 scmd = '''

synapse/tests/test_lib_stormlib_model.py

@@ -1,4 +1,6 @@
 import synapse.exc as s_exc
+import synapse.common as s_common
+
 import synapse.lib.time as s_time
 import synapse.lib.layer as s_layer
 
@@ -827,3 +829,109 @@ class StormlibModelTest(s_test.SynTest):
             with self.raises(s_exc.BadArg) as ectx:
                 await core.nodes('$lib.model.migration.s.riskHasVulnToVulnerable(newp)')
             self.isin('must be a node', ectx.exception.errinfo['mesg'])
+
+    async def test_stormlib_model_migration_s_inet_ssl_to_tls_servercert(self):
+        async with self.getRegrCore('inet_ssl_to_tls_servercert') as core:
+            nodes = await core.nodes('meta:source')
+            self.len(1, nodes)
+
+            nodes = await core.nodes('meta:source -(seen)> *')
+            self.len(3, nodes)
+            for node in nodes:
+                self.eq(node.ndef[0], 'inet:ssl:cert')
+
+            nodes = await core.nodes('inet:ssl:cert')
+            self.len(3, nodes)
+
+            nodes = await core.nodes('file:bytes')
+            self.len(3, nodes)
+
+            nodes = await core.nodes('crypto:x509:cert')
+            self.len(2, nodes)
+
+            nodes = await core.nodes('inet:tls:servercert')
+            self.len(0, nodes)
+
+            q = 'inet:ssl:cert | $lib.model.migration.s.inetSslCertToTlsServerCert($node)'
+            await core.nodes(q)
+
+            nodes = await core.nodes('file:bytes')
+            self.len(3, nodes)
+
+            nodes = await core.nodes('crypto:x509:cert')
+            self.len(3, nodes)
+
+            nodes = await core.nodes('inet:tls:servercert')
+            self.len(3, nodes)
+
+            nodes = await core.nodes('crypto:x509:cert=(cert1,)')
+            self.len(1, nodes)
+            cert1 = nodes[0]
+
+            nodes = await core.nodes('inet:tls:servercert:server="tcp://1.2.3.4:443"')
+            self.len(1, nodes)
+            self.eq(nodes[0].get('.seen'), (1688947200000, 1688947200001))
+            self.eq(nodes[0].get('server'), 'tcp://1.2.3.4:443')
+            self.eq(nodes[0].get('cert'), cert1.ndef[1])
+            self.isin('ssl.migration.one', nodes[0].tags)
+
+            nodes = await core.nodes('crypto:x509:cert=(cert2,)')
+            self.len(1, nodes)
+            cert2 = nodes[0]
+
+            nodes = await core.nodes('inet:tls:servercert:server="tcp://[fe80::1]:8080"')
+            self.len(1, nodes)
+            self.none(nodes[0].get('.seen'))
+            self.eq(nodes[0].get('server'), 'tcp://[fe80::1]:8080')
+            self.eq(nodes[0].get('cert'), cert2.ndef[1])
+            self.isin('ssl.migration.two', nodes[0].tags)
+
+            sha256 = 'aa0366ffb013ba2053e45cd7e4bcc8acd6a6c1bafc82eddb4e155876734c5e25'
+            opts = {'vars': {'sha256': sha256}}
+
+            nodes = await core.nodes('file:bytes=$sha256', opts=opts)
+            self.len(1, nodes)
+            file = nodes[0]
+
+            # This cert was created by the migration code so do a little extra
+            # checking
+            nodes = await core.nodes('crypto:x509:cert:file=$sha256', opts=opts)
+            self.len(1, nodes)
+            self.eq(nodes[0].get('file'), file.ndef[1])
+            self.eq(nodes[0].ndef, ('crypto:x509:cert', s_common.guid(sha256)))
+            cert3 = nodes[0]
+
+            nodes = await core.nodes('inet:tls:servercert:server="tcp://8.8.8.8:53" $node.data.load(foo)')
+            self.len(1, nodes)
+            self.none(nodes[0].get('.seen'))
+            self.eq(nodes[0].get('server'), 'tcp://8.8.8.8:53')
+            self.eq(nodes[0].get('cert'), cert3.ndef[1])
+            self.isin('ssl.migration.three', nodes[0].tags)
+            self.eq(nodes[0].nodedata, {'foo': 'bar'})
+
+            # Check that edges were migrated
+            nodes = await core.nodes('meta:source -(seen)> *')
+            self.len(6, nodes)
+            self.sorteq(
+                [k.ndef[0] for k in nodes],
+                (
+                    'inet:ssl:cert', 'inet:ssl:cert', 'inet:ssl:cert',
+                    'inet:tls:servercert', 'inet:tls:servercert', 'inet:tls:servercert',
+                )
+            )
+
+            with self.raises(s_exc.BadArg) as exc:
+                await core.callStorm('inet:server | $lib.model.migration.s.inetSslCertToTlsServerCert($node)')
+            self.isin(', not inet:server', exc.exception.get('mesg'))
+
+        async with self.getRegrCore('inet_ssl_to_tls_servercert') as core:
+            q = 'inet:ssl:cert | $lib.model.migration.s.inetSslCertToTlsServerCert($node, nodata=$lib.true)'
+            await core.nodes(q)
+
+            nodes = await core.nodes('inet:tls:servercert:server="tcp://8.8.8.8:53" $node.data.load(foo)')
+            self.len(1, nodes)
+            self.none(nodes[0].get('.seen'))
+            self.eq(nodes[0].get('server'), 'tcp://8.8.8.8:53')
+            self.eq(nodes[0].get('cert'), cert3.ndef[1])
+            self.isin('ssl.migration.three', nodes[0].tags)
+            self.eq(nodes[0].nodedata, {'foo': None})

synapse/tests/test_lib_stormlib_modelext.py

@@ -22,6 +22,9 @@ class StormtypesModelextTest(s_test.SynTest):
 
                 $pinfo = ({"doc": "Extended a core model."})
                 $lib.model.ext.addFormProp(test:int, _tick, (time, ({})), $propinfo)
+
+                $edgeinfo = ({"doc": "A test edge."})
+                $lib.model.ext.addEdge(inet:user, _copies, *, $edgeinfo)
             ''')
 
             nodes = await core.nodes('[ _visi:int=10 :tick=20210101 ._woot=30 +#lol:score=99 ]')
@@ -44,6 +47,12 @@ class StormtypesModelextTest(s_test.SynTest):
                 q = '''$lib.model.ext.addUnivProp(_woot, (time, ({})), ({}))'''
                 await core.callStorm(q)
 
+            with self.raises(s_exc.DupEdgeType):
+                q = '''$lib.model.ext.addEdge(inet:user, _copies, *, ({}))'''
+                await core.callStorm(q)
+
+            self.nn(core.model.edge(('inet:user', '_copies', None)))
+
             # Grab the extended model definitions
             model_defs = await core.callStorm('return ( $lib.model.ext.getExtModel() )')
             self.isinstance(model_defs, dict)
@@ -55,6 +64,7 @@ class StormtypesModelextTest(s_test.SynTest):
                 $lib.model.ext.delFormProp(_visi:int, tick)
                 $lib.model.ext.delFormProp(test:int, _tick)
                 $lib.model.ext.delForm(_visi:int)
+                $lib.model.ext.delEdge(inet:user, _copies, *)
             ''')
 
             self.none(core.model.form('_visi:int'))
@@ -62,6 +72,7 @@ class StormtypesModelextTest(s_test.SynTest):
             self.none(core.model.prop('_visi:int:tick'))
             self.none(core.model.prop('test:int:_tick'))
             self.none(core.model.tagprop('score'))
+            self.none(core.model.edge(('inet:user', '_copies', None)))
 
             # Underscores can exist in extended names but only at specific locations
             q = '''$l =$lib.list('str', ({})) $d=({"doc": "Foo"})
@@ -105,6 +116,26 @@ class StormtypesModelextTest(s_test.SynTest):
                 q = '''$lib.model.ext.addTagProp(_someones:_score^value, (int, ({})), ({}))'''
                 await core.callStorm(q)
 
+            with self.raises(s_exc.BadEdgeDef):
+                q = '''$lib.model.ext.addEdge(*, does, *, ({}))'''
+                await core.callStorm(q)
+
+            with self.raises(s_exc.BadEdgeDef):
+                q = '''$lib.model.ext.addEdge(*, _NEWP, *, ({}))'''
+                await core.callStorm(q)
+
+            with self.raises(s_exc.BadEdgeDef):
+                q = '''$lib.model.ext.addEdge(*, "_ne wp", *, ({}))'''
+                await core.callStorm(q)
+
+            with self.raises(s_exc.BadEdgeDef):
+                q = f'''$lib.model.ext.addEdge(*, "_{'a'*201}", *, ({{}}))'''
+                await core.callStorm(q)
+
+            with self.raises(s_exc.BadEdgeDef):
+                q = '''$lib.model.ext.delEdge(*, "_ne wp", *)'''
+                await core.callStorm(q)
+
             # Permission errors
             visi = await core.auth.addUser('visi')
             opts = {'user': visi.iden}
@@ -133,6 +164,10 @@ class StormtypesModelextTest(s_test.SynTest):
                     $lib.model.ext.addTagProp(score, (int, ({})), $tagpropinfo)
                 ''', opts=opts)
 
+            with self.raises(s_exc.AuthDeny):
+                q = '''$lib.model.ext.addEdge(*, _does, *, ({}))'''
+                await core.callStorm(q, opts=opts)
+
         # Reload the model extensions automatically
         async with self.getTestCore() as core:
             opts = {'vars': {'model_defs': model_defs}}
@@ -174,6 +209,9 @@ class StormtypesModelextTest(s_test.SynTest):
 
                 $pinfo = ({"doc": "NEWP"})
                 $lib.model.ext.addFormProp(test:int, _tick, (time, ({})), $propinfo)
+
+                $edgeinfo = ({"doc": "NEWP"})
+                $lib.model.ext.addEdge(inet:user, _copies, *, $edgeinfo)
             ''')
 
             q = '''return ($lib.model.ext.addExtModel($model_defs))'''
@@ -196,6 +234,11 @@ class StormtypesModelextTest(s_test.SynTest):
                 opts = {'vars': {'model_defs': {'univs': model_defs['univs']}}}
                 await core.callStorm(q, opts)
 
+            q = '''return ($lib.model.ext.addExtModel($model_defs))'''
+            with self.raises(s_exc.BadEdgeDef) as cm:
+                opts = {'vars': {'model_defs': {'edges': model_defs['edges']}}}
+                await core.callStorm(q, opts)
+
         # Reload the model extensions from the dump by hand
         async with self.getTestCore() as core:
             opts = {'vars': {'model_defs': model_defs}}
@@ -212,6 +255,10 @@ class StormtypesModelextTest(s_test.SynTest):
             for ($prop, $def, $info) in $model_defs.univs {
                 $lib.model.ext.addUnivProp($prop, $def, $info)
             }
+            for ($edge, $info) in $model_defs.edges {
+                ($n1form, $verb, $n2form) = $edge
+                $lib.model.ext.addEdge($n1form, $verb, $n2form, $info)
+            }
             '''
             await core.nodes(q, opts)
 
@@ -227,6 +274,8 @@ class StormtypesModelextTest(s_test.SynTest):
             self.eq(nodes[0].ndef, ('test:int', 1234))
             self.eq(nodes[0].get('_tick'), 1609459200000)
 
+            self.nn(core.model.edge(('inet:user', '_copies', None)))
+
     async def test_lib_stormlib_behold_modelext(self):
         self.skipIfNexusReplay()
         async with self.getTestCore() as core:
@@ -252,6 +301,7 @@ class StormtypesModelextTest(s_test.SynTest):
                         $lib.model.ext.addFormProp(_behold:score, rank, (int, ({})), ({"doc": "second string"}))
                         $lib.model.ext.addUnivProp(_beep, (int, ({})), ({"doc": "third string"}))
                         $lib.model.ext.addTagProp(thingy, (int, ({})), ({"doc": "fourth string"}))
+                        $lib.model.ext.addEdge(*, _goes, geo:place, ({"doc": "fifth string"}))
                     ''')
 
                     formmesg = await sock.receive_json()
@@ -279,11 +329,17 @@ class StormtypesModelextTest(s_test.SynTest):
                     self.eq(tagpmesg['data']['info']['name'], 'thingy')
                     self.eq(tagpmesg['data']['info']['info'], {'doc': 'fourth string'})
 
+                    edgemesg = await sock.receive_json()
+                    self.eq(edgemesg['data']['event'], 'model:edge:add')
+                    self.eq(edgemesg['data']['info']['edge'], (None, '_goes', 'geo:place'))
+                    self.eq(edgemesg['data']['info']['info'], {'doc': 'fifth string'})
+
                     await core.callStorm('''
                         $lib.model.ext.delTagProp(thingy)
                         $lib.model.ext.delUnivProp(_beep)
                         $lib.model.ext.delFormProp(_behold:score, rank)
                         $lib.model.ext.delForm(_behold:score)
+                        $lib.model.ext.delEdge(*, _goes, geo:place)
                     ''')
                     deltagp = await sock.receive_json()
                     self.eq(deltagp['data']['event'], 'model:tagprop:del')
@@ -302,6 +358,10 @@ class StormtypesModelextTest(s_test.SynTest):
                     self.eq(delform['data']['event'], 'model:form:del')
                     self.eq(delform['data']['info']['form'], '_behold:score')
 
+                    deledge = await sock.receive_json()
+                    self.eq(deledge['data']['event'], 'model:edge:del')
+                    self.eq(deledge['data']['info']['edge'], (None, '_goes', 'geo:place'))
+
     async def test_lib_stormlib_modelext_delform(self):
         '''
         Verify extended forms can't be deleted if they have associated extended props
@@ -385,6 +445,10 @@ class StormtypesModelextTest(s_test.SynTest):
                 '$lib.model.ext.addTagProp(_foo:bar, (), ())',
                 'Tag property definitions should be a dict.'
             ),
+            (
+                '$lib.model.ext.addEdge(*, _foo, *, ())',
+                'Edge info should be a dict.'
+            ),
         )
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_stormlib_smtp.py

@@ -1,3 +1,6 @@
+import ssl
+import email.mime.multipart as e_muiltipart
+
 from unittest import mock
 import synapse.tests.utils as s_test
 
@@ -5,7 +8,9 @@ class SmtpTest(s_test.SynTest):
 
     async def test_storm_smtp(self):
 
+        called_args = []
         async def send(*args, **kwargs):
+            called_args.append((args, kwargs))
             return
 
         with mock.patch('aiosmtplib.send', send):
@@ -14,6 +19,9 @@ class SmtpTest(s_test.SynTest):
 
                 retn = await core.callStorm('return($lib.inet.smtp.message().send(127.0.0.1))')
                 self.false(retn[0])
+                self.eq(retn[1].get('err'), 'StormRuntimeError')
+                self.isin('The inet:smtp:message has no HTML or text body.', retn[1].get('errmsg'))
+                self.len(0, called_args)
 
                 retn = await core.callStorm('''
                     $message = $lib.inet.smtp.message()
@@ -31,6 +39,49 @@ class SmtpTest(s_test.SynTest):
                     return($message.send('smtp.gmail.com', port=465, usetls=true))
                 ''')
                 self.eq(retn, (True, {}))
+                mesg = called_args[-1][0][0]  # type: e_muiltipart.MIMEMultipart
+                self.eq(mesg.get_all('subject'), ['woot'])
+                payload = mesg.get_payload()
+                self.len(2, payload)
+                self.eq([pl.get_content_type() for pl in payload],
+                        ['text/plain', 'text/html'])
+                ctx = self.nn(called_args[-1][1].get('tls_context'))  # type: ssl.SSLContext
+                self.eq(ctx.verify_mode, ssl.CERT_REQUIRED)
+                self.eq(called_args[-1][1].get('port'), 465)
+                self.true(called_args[-1][1].get('use_tls'))
+                self.false(called_args[-1][1].get('start_tls'))
+
+                retn = await core.callStorm('''
+                    $message = $lib.inet.smtp.message()
+                    $message.text = "HELLO WORLD"
+                    $message.sender = visi@vertex.link
+                    $message.headers.Subject = woot
+                    $message.recipients.append(visi@vertex.link)
+                    return($message.send('smtp.gmail.com', port=465, starttls=true, ssl_verify=(false)))
+                ''')
+                self.eq(retn, (True, {}))
+                mesg = called_args[-1][0][0]  # type: e_muiltipart.MIMEMultipart
+                payload = mesg.get_payload()
+                self.len(1, payload)
+                self.eq([pl.get_content_type() for pl in payload], ['text/plain'])
+                ctx = self.nn(called_args[-1][1].get('tls_context'))  # type: ssl.SSLContext
+                self.eq(ctx.verify_mode, ssl.CERT_NONE)
+                self.false(called_args[-1][1].get('use_tls'))
+                self.true(called_args[-1][1].get('start_tls'))
+
+                retn = await core.callStorm('''
+                    $message = $lib.inet.smtp.message()
+                    $message.text = "HELLO WORLD"
+                    $message.sender = visi@vertex.link
+                    $message.headers.Subject = woot
+                    $message.recipients.append(visi@vertex.link)
+                    return($message.send('smtp.gmail.com', port=25))
+                ''')
+                self.eq(retn, (True, {}))
+                self.none(called_args[-1][1].get('tls_context'))  # type: ssl.SSLContext
+                self.eq(called_args[-1][1].get('port'), 25)
+                self.false(called_args[-1][1].get('use_tls'))
+                self.false(called_args[-1][1].get('start_tls'))
 
                 isok, info = await core.callStorm('''
                     $message = $lib.inet.smtp.message()