synapse 2.152.0__py311-none-any.whl → 2.154.0__py311-none-any.whl

synapse/models/risk.py

@@ -99,9 +99,22 @@ class RiskModule(s_module.CoreModule):
                 }),
 
                 ('risk:threat:type:taxonomy', ('taxonomy', {}), {
-                    'doc': 'A taxonomy of threat types.',
                     'interfaces': ('taxonomy',),
-                }),
+                    'doc': 'A taxonomy of threat types.'}),
+
+                ('risk:leak', ('guid', {}), {
+                    'doc': 'An event where information was disclosed without permission.'}),
+
+                ('risk:leak:type:taxonomy', ('taxonomy', {}), {
+                    'interfaces': ('taxonomy',),
+                    'doc': 'A taxonomy of leak event types.'}),
+
+                ('risk:extortion', ('guid', {}), {
+                    'doc': 'An event where an attacker attempted to extort a victim.'}),
+
+                ('risk:extortion:type:taxonomy', ('taxonomy', {}), {
+                    'interfaces': ('taxonomy',),
+                    'doc': 'A taxonomy of extortion event types.'}),
             ),
             'edges': (
                 # some explicit examples...
@@ -141,6 +154,12 @@ class RiskModule(s_module.CoreModule):
                     'doc': 'The target node was stolen or copied as a result of the compromise.'}),
                 (('risk:mitigation', 'addresses', 'ou:technique'), {
                     'doc': 'The mitigation addresses the technique.'}),
+
+                (('risk:leak', 'leaked', None), {
+                    'doc': 'The leak included the disclosure of the target node.'}),
+
+                (('risk:extortion', 'leveraged', None), {
+                    'doc': 'The extortion event was based on attacker access to the target node.'}),
             ),
             'forms': (
 
@@ -596,6 +615,9 @@ class RiskModule(s_module.CoreModule):
 
                     ('ext:id', ('str', {}), {
                         'doc': 'An external identifier for the alert.'}),
+
+                    ('host', ('it:host', {}), {
+                        'doc': 'The host which generated the alert.'}),
                 )),
                 ('risk:compromisetype', {}, ()),
                 ('risk:compromise', {}, (
@@ -612,6 +634,12 @@ class RiskModule(s_module.CoreModule):
                     ('reporter:name', ('ou:name', {}), {
                         'doc': 'The name of the organization reporting on the compromise.'}),
 
+                    ('ext:id', ('str', {}), {
+                        'doc': 'An external unique ID for the compromise.'}),
+
+                    ('url', ('inet:url', {}), {
+                        'doc': 'A URL which documents the compromise.'}),
+
                     ('type', ('risk:compromisetype', {}), {
                         'ex': 'cno.breach',
                         'doc': 'A type for the compromise, as a taxonomy entry.'}),
@@ -815,6 +843,102 @@ class RiskModule(s_module.CoreModule):
                         'doc': 'An external unique ID for the attack.'}),
 
                 )),
+
+                ('risk:leak:type:taxonomy', {}, ()),
+                ('risk:leak', {}, (
+
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'A simple name for the leak event.'}),
+
+                    ('desc', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A description of the leak event.'}),
+
+                    ('reporter', ('ou:org', {}), {
+                        'doc': 'The organization reporting on the leak event.'}),
+
+                    ('reporter:name', ('ou:name', {}), {
+                        'doc': 'The name of the organization reporting on the leak event.'}),
+
+                    ('disclosed', ('time', {}), {
+                        'doc': 'The time the leaked information was disclosed.'}),
+
+                    ('owner', ('ps:contact', {}), {
+                        'doc': 'The owner of the leaked information.'}),
+
+                    ('leaker', ('ps:contact', {}), {
+                        'doc': 'The identity which leaked the information.'}),
+
+                    ('type', ('risk:leak:type:taxonomy', {}), {
+                        'doc': 'A type taxonomy for the leak.'}),
+
+                    ('goal', ('ou:goal', {}), {
+                        'doc': 'The goal of the leaker in disclosing the information.'}),
+
+                    ('compromise', ('risk:compromise', {}), {
+                        'doc': 'The compromise which allowed the leaker access to the information.'}),
+
+                    ('public', ('bool', {}), {
+                        'doc': 'Set to true if the leaked information was made publicly available.'}),
+
+                    ('public:url', ('inet:url', {}), {
+                        'doc': 'The URL where the leaked information was made publicly available.'}),
+
+                )),
+
+                ('risk:extortion:type:taxonomy', {}, ()),
+                ('risk:extortion', {}, (
+
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'A name for the extortion event.'}),
+
+                    ('desc', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A description of the extortion event.'}),
+
+                    ('reporter', ('ou:org', {}), {
+                        'doc': 'The organization reporting on the extortion event.'}),
+
+                    ('reporter:name', ('ou:name', {}), {
+                        'doc': 'The name of the organization reporting on the extortion event.'}),
+
+                    ('demanded', ('time', {}), {
+                        'doc': 'The time that the attacker made their demands.'}),
+
+                    ('goal', ('ou:goal', {}), {
+                        'doc': 'The goal of the attacker in extorting the victim.'}),
+
+                    ('type', ('risk:extortion:type:taxonomy', {}), {
+                        'doc': 'A type taxonomy for the extortion event.'}),
+
+                    ('attacker', ('ps:contact', {}), {
+                        'doc': 'The extortion attacker identity.'}),
+
+                    ('target', ('ps:contact', {}), {
+                        'doc': 'The extortion target identity.'}),
+
+                    ('success', ('bool', {}), {
+                        'doc': 'Set to true if the victim met the attackers demands.'}),
+
+                    ('enacted', ('bool', {}), {
+                        'doc': 'Set to true if attacker carried out the threat.'}),
+
+                    ('public', ('bool', {}), {
+                        'doc': 'Set to true if the attacker publicly announced the extortion.'}),
+
+                    ('public:url', ('inet:url', {}), {
+                        'doc': 'The URL where the attacker publicly announced the extortion.'}),
+
+                    ('compromise', ('risk:compromise', {}), {
+                        'doc': 'The compromise which allowed the attacker to extort the target.'}),
+
+                    ('demanded:payment:price', ('econ:price', {}), {
+                        'doc': 'The payment price which was demanded.'}),
+
+                    ('demanded:payment:currency', ('econ:currency', {}), {
+                        'doc': 'The currency in which payment was demanded.'}),
+
+                )),
             ),
         }
         name = 'risk'

synapse/models/syn.py

@@ -236,6 +236,12 @@ class SynModule(s_module.CoreModule):
                     ('form', ('str', {'lower': True, 'strip': True}), {
                         'doc': 'Form the trigger is watching for.'
                     }),
+                    ('verb', ('str', {'lower': True, 'strip': True}), {
+                        'doc': 'Edge verb the trigger is watching for.'
+                    }),
+                    ('n2form', ('str', {'lower': True, 'strip': True}), {
+                        'doc': 'N2 form the trigger is watching for.'
+                    }),
                     ('prop', ('str', {'lower': True, 'strip': True}), {
                         'doc': 'Property the trigger is watching for.'
                     }),

synapse/tests/files/stormpkg/badapidef.yaml

@@ -0,0 +1,13 @@
+name: testpkg
+version: 0.0.1
+
+modules:
+  - name: apimod
+    apidefs:
+      - name: status
+        desc: Get the status of the foo.
+        type:
+          type: function
+          returns:
+            type: 42
+            desc: A status dictionary.

synapse/tests/files/stormpkg/storm/modules/apimod

@@ -0,0 +1,10 @@
+
+function search(text, mintime="-30days") {
+    for $t in $text {
+        [ it:devstr=$t ]
+    }
+}
+
+function status() {
+    return(({"ok": true}))
+}

synapse/tests/files/stormpkg/testpkg.yaml

@@ -7,6 +7,29 @@ logo:
 
 modules:
     - name: testmod
+    - name: apimod
+      apidefs:
+        - name: search
+          desc: |
+            Execute a search
+            
+            This API will foo the bar.
+          type:
+            type: function
+            args:
+              - { name: text, type: str, desc: "The text." }
+              - { name: mintime, type: [str, int], desc: "The mintime.", default: "-30days" }
+            returns:
+              name: yields
+              type: node
+              desc: Yields it:dev:str nodes.
+        - name: status
+          desc: Get the status of the foo.
+          type:
+            type: function
+            returns:
+              type: dict
+              desc: A status dictionary.
 
 external_modules:
     - name: testext

synapse/tests/test_axon.py

@@ -278,8 +278,14 @@ class AxonTest(s_t_utils.SynTest):
         (lsize, l256) = await axon.put(linesbuf)
         (jsize, j256) = await axon.put(jsonsbuf)
         (bsize, b256) = await axon.put(b'\n'.join((jsonsbuf, linesbuf)))
+        (binsize, bin256) = await axon.put(bin_buf)
+
         lines = [item async for item in axon.readlines(s_common.ehex(l256))]
         self.eq(('asdf', '', 'qwer'), lines)
+        lines = [item async for item in axon.readlines(s_common.ehex(bin256))]  # Default is errors=ignore
+        self.eq(lines, ['/$A\x00_v4\x1b'])
+        lines = [item async for item in axon.readlines(s_common.ehex(bin256), errors='replace')]
+        self.eq(lines, ['�/$�A�\x00_�v4��\x1b'])
         jsons = [item async for item in axon.jsonlines(s_common.ehex(j256))]
         self.eq(({'foo': 'bar'}, {'baz': 'faz'}), jsons)
         jsons = []
@@ -288,9 +294,8 @@ class AxonTest(s_t_utils.SynTest):
                 jsons.append(item)
         self.eq(({'foo': 'bar'}, {'baz': 'faz'}), jsons)
 
-        binsize, bin256 = await axon.put(bin_buf)
         with self.raises(s_exc.BadDataValu):
-            lines = [item async for item in axon.readlines(s_common.ehex(bin256))]
+            lines = [item async for item in axon.readlines(s_common.ehex(bin256), errors=None)]
 
         with self.raises(s_exc.NoSuchFile):
             lines = [item async for item in axon.readlines(s_common.ehex(newphash))]

synapse/tests/test_cortex.py

@@ -2046,8 +2046,10 @@ class CortexTest(s_t_utils.SynTest):
                 await core.nodes('test:str +test:str@=2018')
             with self.raises(s_exc.NoSuchCmpr):
                 await core.nodes('test:str +test:str:tick*near=newp')
-            with self.raises(s_exc.BadTypeValu):
+            with self.raises(s_exc.NoSuchCmpr):
                 await core.nodes('test:str +#test*near=newp')
+            with self.raises(s_exc.BadTypeValu):
+                await core.nodes('test:str +#test*in=newp')
             with self.raises(s_exc.BadSyntax):
                 await core.nodes('test:str -> # } limit 10')
             with self.raises(s_exc.BadSyntax):
@@ -6115,55 +6117,71 @@ class CortexBasicTest(s_t_utils.SynTest):
             self.true(layr.lockmemory)
 
     async def test_cortex_storm_lib_dmon(self):
-        async with self.getTestCoreAndProxy() as (core, prox):
-            nodes = await core.nodes('''
 
-                $lib.print(hi)
+        with self.getTestDir() as dirn:
 
-                $tx = $lib.queue.add(tx)
-                $rx = $lib.queue.add(rx)
+            async with self.getTestCoreAndProxy(dirn=dirn) as (core, prox):
+                nodes = await core.nodes('''
 
-                $ddef = $lib.dmon.add(${
+                    $lib.print(hi)
 
-                    $rx = $lib.queue.get(tx)
-                    $tx = $lib.queue.get(rx)
+                    $tx = $lib.queue.add(tx)
+                    $rx = $lib.queue.add(rx)
 
-                    $ipv4 = nope
-                    for ($offs, $ipv4) in $rx.gets(wait=1) {
-                        [ inet:ipv4=$ipv4 ]
-                        $rx.cull($offs)
-                        $tx.put($ipv4)
-                    }
-                })
+                    $ddef = $lib.dmon.add(${
 
-                $tx.put(1.2.3.4)
+                        $rx = $lib.queue.get(tx)
+                        $tx = $lib.queue.get(rx)
 
-                for ($xoff, $xpv4) in $rx.gets(size=1, wait=1) { }
+                        $ipv4 = nope
+                        for ($offs, $ipv4) in $rx.gets(wait=1) {
+                            [ inet:ipv4=$ipv4 ]
+                            $rx.cull($offs)
+                            $tx.put($ipv4)
+                        }
+                    })
 
-                $lib.print(xed)
+                    $tx.put(1.2.3.4)
 
-                inet:ipv4=$xpv4
+                    for ($xoff, $xpv4) in $rx.gets(size=1, wait=1) { }
 
-                $lib.dmon.del($ddef.iden)
+                    $lib.print(xed)
 
-                $lib.queue.del(tx)
-                $lib.queue.del(rx)
-            ''')
-            self.len(1, nodes)
-            self.len(0, await prox.getStormDmons())
+                    inet:ipv4=$xpv4
 
-            with self.raises(s_exc.NoSuchIden):
-                await core.nodes('$lib.dmon.del(newp)')
+                    $lib.dmon.del($ddef.iden)
 
-            await core.stormlist('auth.user.add user')
-            user = await core.auth.getUserByName('user')
-            asuser = {'user': user.iden}
+                    $lib.queue.del(tx)
+                    $lib.queue.del(rx)
+                ''')
+                self.len(1, nodes)
+                self.len(0, await prox.getStormDmons())
 
-            ddef = await core.callStorm('return($lib.dmon.add(${$lib.print(foo)}))')
-            iden = ddef.get('iden')
+                with self.raises(s_exc.NoSuchIden):
+                    await core.nodes('$lib.dmon.del(newp)')
 
-            with self.raises(s_exc.AuthDeny):
-                await core.callStorm(f'$lib.dmon.del({iden})', opts=asuser)
+                await core.stormlist('auth.user.add user')
+                user = await core.auth.getUserByName('user')
+                asuser = {'user': user.iden}
+
+                ddef = await core.callStorm('return($lib.dmon.add(${$lib.print(foo)}))')
+                iden = ddef.get('iden')
+                asuser['vars'] = {'iden': iden}
+
+                with self.raises(s_exc.AuthDeny):
+                    await core.callStorm(f'$lib.dmon.del($iden)', opts=asuser)
+
+                # remove the dmon without a nexus entry to verify recover works
+                await core._delStormDmon(iden)
+                self.none(await core.callStorm('return($lib.dmon.get($iden))', opts=asuser))
+                self.eq('storm:dmon:add', (await core.nexsroot.nexslog.last())[1][1])
+
+            async with self.getTestCoreAndProxy(dirn=dirn) as (core, prox):
+
+                # nexus recover() previously failed on adding to the hive
+                # although the dmon would get successfully started
+                self.nn(await core.callStorm('return($lib.dmon.get($iden))', opts=asuser))
+                self.nn(core.stormdmonhive.get(iden))
 
     async def test_cortex_storm_dmon_view(self):
 
@@ -7164,6 +7182,66 @@ class CortexBasicTest(s_t_utils.SynTest):
             # Avoid races in cleanup
             del genr
 
+    async def test_cortex_syncnodeedits(self):
+
+        async with self.getTestCore() as core:
+
+            layr00 = core.getLayer().iden
+            layr01 = (await core.addLayer())['iden']
+            view01 = (await core.addView({'layers': (layr01,)}))['iden']
+
+            async def layrgenr(layr, startoff, endoff=None, newlayer=False):
+                wait = endoff is None
+                async for ioff, item, meta in layr.syncNodeEdits2(startoff, wait=wait):
+                    if endoff is not None and ioff >= endoff:
+                        break
+                    yield ioff, item, meta
+
+            indx = await core.getNexsIndx()
+
+            offsdict = {
+                layr00: indx,
+                layr01: indx,
+            }
+
+            genr = None
+
+            try:
+
+                # test that a slow consumer can continue to stream edits
+                # even if a layer exceeds the window maxsize
+
+                oldv = s_layer.WINDOW_MAXSIZE
+                s_layer.WINDOW_MAXSIZE = 2
+
+                genr = core._syncNodeEdits(offsdict, layrgenr, wait=True)
+
+                nodes = await core.nodes('[ test:str=foo ]')
+                item = await asyncio.wait_for(genr.__anext__(), timeout=2)
+                self.eq(s_common.uhex(nodes[0].iden()), item[1][0][0])
+
+                # we should now be in live sync
+                # and the empty layer will be pulling from the window
+
+                nodes = await core.nodes('[ test:str=bar ]')
+                item = await asyncio.wait_for(genr.__anext__(), timeout=2)
+                self.eq(s_common.uhex(nodes[0].iden()), item[1][0][0])
+
+                # add more nodes than the window size without consuming from the genr
+
+                opts = {'view': view01}
+                nodes = await core.nodes('for $s in (baz, bam, cat, dog) { [ test:str=$s ] }', opts=opts)
+                items = [await asyncio.wait_for(genr.__anext__(), timeout=2) for _ in range(4)]
+                self.sorteq(
+                    [s_common.uhex(n.iden()) for n in nodes],
+                    [item[1][0][0] for item in items],
+                )
+
+            finally:
+                s_layer.WINDOW_MAXSIZE = oldv
+                if genr is not None:
+                    del genr
+
     async def test_cortex_all_layr_read(self):
         async with self.getTestCore() as core:
             layr = core.getView().layers[0].iden
@@ -7613,3 +7691,121 @@ class CortexBasicTest(s_t_utils.SynTest):
                 self.isin('Set admin=True for lowuser', mesg.get('message'))
                 self.eq('admin', mesg.get('username'))
                 self.eq('lowuser', mesg.get('target_username'))
+
+    async def test_cortex_ext_httpapi(self):
+        # Cortex API tests for Extended HttpAPI
+        async with self.getTestCore() as core:  # type: s_cortex.Cortex
+
+            newp = s_common.guid()
+            with self.raises(s_exc.SynErr):
+                await core.setHttpApiIndx(newp, 0)
+
+            unfo = await core.getUserDefByName('root')
+            view = core.getView()
+            info = await core.addHttpExtApi({
+                'path': 'test/path/(hehe|haha)/(.*)',
+                'owner': unfo.get('iden'),
+                'view': view.iden,
+            })
+
+            info2 = await core.addHttpExtApi({
+                'path': 'something/else',
+                'owner': unfo.get('iden'),
+                'view': view.iden,
+            })
+
+            info3 = await core.addHttpExtApi({
+                'path': 'something/else/goes/here',
+                'owner': unfo.get('iden'),
+                'view': view.iden,
+            })
+
+            info4 = await core.addHttpExtApi({
+                'path': 'another/item',
+                'owner': unfo.get('iden'),
+                'view': view.iden,
+            })
+
+            iden = info.get('iden')
+
+            adef = await core.getHttpExtApi(iden)
+            self.eq(adef, info)
+
+            adef, args = await core.getHttpExtApiByPath('test/path/hehe/wow')
+            self.eq(adef, info)
+            self.eq(args, ('hehe', 'wow'))
+
+            adef, args = await core.getHttpExtApiByPath('test/path/hehe/wow/more/')
+            self.eq(adef, info)
+            self.eq(args, ('hehe', 'wow/more/'))
+
+            adef, args = await core.getHttpExtApiByPath('test/path/HeHe/wow')
+            self.none(adef)
+            self.eq(args, ())
+
+            async with core.getLocalProxy() as prox:
+                adef, args = await prox.getHttpExtApiByPath('test/path/haha/words')
+                self.eq(adef, info)
+                self.eq(args, ('haha', 'words'))
+
+            self.len(4, core._exthttpapicache)
+
+            # Reordering / safety
+            self.eq(1, await core.setHttpApiIndx(info4.get('iden'), 1))
+
+            # Cache is cleared when reloading
+            self.len(0, core._exthttpapicache)
+            adef, args = await core.getHttpExtApiByPath('test/path/hehe/wow')
+            self.eq(adef, info)
+            self.len(1, core._exthttpapicache)
+
+            self.eq([adef.get('iden') for adef in await core.getHttpExtApis()],
+                    [info.get('iden'), info4.get('iden'), info2.get('iden'), info3.get('iden')])
+
+            items = await core.getHttpExtApis()
+            self.eq(items, (info, info4, info2, info3))
+
+            # Tiny sleep to ensure that updated ticks forward when modified
+            created = adef.get('created')
+            updated = adef.get('updated')
+            await asyncio.sleep(0.005)
+            adef = await core.modHttpExtApi(iden, 'name', 'wow')
+            self.eq(adef.get('created'), created)
+            self.gt(adef.get('updated'), updated)
+
+            # Sad path
+
+            with self.raises(s_exc.SynErr):
+                await core.setHttpApiIndx(newp, 0)
+
+            with self.raises(s_exc.BadArg):
+                await core.setHttpApiIndx(newp, -1)
+
+            with self.raises(s_exc.NoSuchUser):
+                await core.modHttpExtApi(iden, 'owner', newp)
+
+            with self.raises(s_exc.NoSuchView):
+                await core.modHttpExtApi(iden, 'view', newp)
+
+            with self.raises(s_exc.BadArg):
+                await core.modHttpExtApi(iden, 'created', 1234)
+
+            with self.raises(s_exc.BadArg):
+                await core.modHttpExtApi(iden, 'updated', 1234)
+
+            with self.raises(s_exc.BadArg):
+                await core.modHttpExtApi(iden, 'creator', s_common.guid())
+
+            with self.raises(s_exc.BadArg):
+                await core.modHttpExtApi(iden, 'newp', newp)
+
+            with self.raises(s_exc.NoSuchIden):
+                await core.modHttpExtApi(newp, 'path', 'a/new/path/')
+
+            with self.raises(s_exc.NoSuchIden):
+                await core.getHttpExtApi(newp)
+
+            self.none(await core.delHttpExtApi(newp))
+
+            with self.raises(s_exc.BadArg):
+                await core.delHttpExtApi('notAGuid')