synapse 2.216.0__tar.gz → 2.218.0__tar.gz

{synapse-2.216.0/synapse.egg-info → synapse-2.218.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.216.0
+Version: 2.218.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0
@@ -22,7 +22,7 @@ Requires-Dist: pyOpenSSL<24.3.0,>=24.0.0
 Requires-Dist: cryptography<44.0.0,>=43.0.1
 Requires-Dist: msgpack<1.2.0,>=1.0.5
 Requires-Dist: xxhash<3.6.0,>=1.4.4
-Requires-Dist: lmdb<1.7.0,>=1.2.1
+Requires-Dist: lmdb<1.8.0,>=1.7.0
 Requires-Dist: tornado<7.0.0,>=6.2.0
 Requires-Dist: regex>=2022.9.11
 Requires-Dist: PyYAML<6.1.0,>=5.4

{synapse-2.216.0 → synapse-2.218.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.216.0'
+version = '2.218.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -26,7 +26,7 @@ dependencies = [
     'cryptography>=43.0.1,<44.0.0',
     'msgpack>=1.0.5,<1.2.0',
     'xxhash>=1.4.4,<3.6.0',
-    'lmdb>=1.2.1,<1.7.0',
+    'lmdb>=1.7.0,<1.8.0',
     'tornado>=6.2.0,<7.0.0',
     'regex>=2022.9.11',
     'PyYAML>=5.4,<6.1.0',

{synapse-2.216.0 → synapse-2.218.0}/synapse/common.py

@@ -17,6 +17,7 @@ import decimal
 import fnmatch
 import hashlib
 import logging
+import tarfile
 import binascii
 import builtins
 import tempfile
@@ -1189,8 +1190,38 @@ def _patch_tornado_json():
     if hasattr(tornado.escape, 'json_decode'):
         tornado.escape.json_decode = s_json.loads
 
+def _patch_tarfile_count():
+    '''
+    Patch tarfile block size reading from the cpython implementation if
+    the interpreter has not been patched for CVE-2025-8194.
+    See https://mail.python.org/archives/list/security-announce@python.org/thread/ZULLF3IZ726XP5EY7XJ7YIN3K5MDYR2D/
+    '''
+    if sys.version_info.major > 3:
+        return
+
+    # Map of minor versions to micro versions which contain the patch
+    min_patched_micros = {
+        11: 14,
+        12: 12,
+        13: 6,
+    }
+    req_micro = min_patched_micros.get(sys.version_info.minor)
+    if req_micro is None:
+        return
+    if sys.version_info.micro >= req_micro:
+        return
+
+    def _block_patched(self, count):
+        if count < 0:  # pragma: no cover
+            raise tarfile.InvalidHeaderError("invalid offset")
+        return _block_patched._orig_block(self, count)
+
+    _block_patched._orig_block = tarfile.TarInfo._block
+    tarfile.TarInfo._block = _block_patched
+
 _patch_http_cookies()
 _patch_tornado_json()
+_patch_tarfile_count()
 
 # TODO:  Switch back to using asyncio.wait_for when we are using py 3.12+
 # This is a workaround for a race where asyncio.wait_for can end up

{synapse-2.216.0 → synapse-2.218.0}/synapse/cortex.py

@@ -5500,6 +5500,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         # TODO Remove the defaults in 3.0.0
         csize = pdef.get('chunk:size', s_const.layer_pdef_csize)
         qsize = pdef.get('queue:size', s_const.layer_pdef_qsize)
+        soffs = pdef.get('offs', -1)
 
         async with await s_base.Base.anit() as base:
 
@@ -5508,7 +5509,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             async def fill():
 
                 try:
-                    filloffs = await self.getStormVar(gvar, -1)
+                    filloffs = await self.getStormVar(gvar, soffs)
                     async for item in layr0.syncNodeEdits(filloffs + 1, wait=True):
                         await queue.put(item)
                     await queue.close()

{synapse-2.216.0 → synapse-2.218.0}/synapse/lib/storm.py

@@ -4906,6 +4906,9 @@ class ViewExecCmd(Cmd):
 
             query = await runt.getStormQuery(text)
             async with runt.getSubRuntime(query, opts=opts) as subr:
+                await subr.enter_context(subr.snap.onWith('print', runt.snap.dist))
+                await subr.enter_context(subr.snap.onWith('warn', runt.snap.dist))
+
                 async for item in subr.execute():
                     await asyncio.sleep(0)
 
@@ -4918,6 +4921,9 @@ class ViewExecCmd(Cmd):
 
             opts = {'view': view}
             async with runt.getSubRuntime(query, opts=opts) as subr:
+                await subr.enter_context(subr.snap.onWith('print', runt.snap.dist))
+                await subr.enter_context(subr.snap.onWith('warn', runt.snap.dist))
+
                 async for item in subr.execute():
                     await asyncio.sleep(0)
 
@@ -5736,6 +5742,9 @@ class RunAsCmd(Cmd):
             opts = {'vars': path.vars}
 
             async with await core.snap(user=user, view=runt.snap.view) as snap:
+                await snap.enter_context(snap.onWith('warn', runt.snap.dist))
+                await snap.enter_context(snap.onWith('print', runt.snap.dist))
+
                 async with await Runtime.anit(query, snap, user=user, opts=opts, root=runt) as subr:
                     subr.debug = runt.debug
                     subr.readonly = runt.readonly
@@ -5758,6 +5767,9 @@ class RunAsCmd(Cmd):
             opts = {'user': user}
 
             async with await core.snap(user=user, view=runt.snap.view) as snap:
+                await snap.enter_context(snap.onWith('warn', runt.snap.dist))
+                await snap.enter_context(snap.onWith('print', runt.snap.dist))
+
                 async with await Runtime.anit(query, snap, user=user, opts=opts, root=runt) as subr:
                     subr.debug = runt.debug
                     subr.readonly = runt.readonly

{synapse-2.216.0 → synapse-2.218.0}/synapse/lib/stormtypes.py

@@ -3653,6 +3653,12 @@ class LibFeed(Lib):
                       {'name': 'data', 'type': 'prim', 'desc': 'Data to send to the ingest function.', },
                   ),
                   'returns': {'type': 'null', }}},
+        {'name': 'fromAxon', 'desc': 'Load a syn.nodes formatted export from axon.',
+         'type': {'type': 'function', '_funcname': '_fromAxon',
+                   'args': (
+                       {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 of the file stored in the axon.', },
+                  ),
+                  'returns': {'type': 'int', 'desc': 'The number of nodes loaded.', }}},
     )
     _storm_lib_path = ('feed',)
 
@@ -7677,7 +7683,23 @@ class Layer(Prim):
 
     @stormfunc(readonly=True)
     async def _methLayerGet(self, name, defv=None):
-        return self.valu.get(name, defv)
+        match name:
+            case 'pushs':
+                pushs = copy.deepcopy(self.valu.get('pushs', {}))
+                for iden, pdef in pushs.items():
+                    gvar = f'push:{iden}'
+                    pdef['offs'] = await self.runt.snap.core.getStormVar(gvar, 0)
+                return pushs
+
+            case 'pulls':
+                pulls = copy.deepcopy(self.valu.get('pulls', {}))
+                for iden, pdef in pulls.items():
+                    gvar = f'pull:{iden}'
+                    pdef['offs'] = await self.runt.snap.core.getStormVar(gvar, 0)
+                return pulls
+
+            case _:
+                return self.valu.get(name, defv)
 
     async def _methLayerSet(self, name, valu):
         name = await tostr(name)

{synapse-2.216.0 → synapse-2.218.0}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 216, 0)
+version = (2, 218, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = '2a44a91a495a7daf2e4e6e44d32c1c3030db3fd2'
+commit = '69efe6d3b57074a6141c75bccafaf8521838dee2'

{synapse-2.216.0 → synapse-2.218.0}/synapse/models/inet.py

@@ -108,6 +108,10 @@ def getAddrScope(ipv6):
 
 class Addr(s_types.Str):
 
+    protos = ('tcp', 'udp', 'icmp', 'host', 'gre')
+    # TODO: this should include icmp and host but requires a migration
+    noports = ('gre',)
+
     def postTypeInit(self):
         s_types.Str.postTypeInit(self)
         self.setNormFunc(str, self._normPyStr)
@@ -140,9 +144,11 @@ class Addr(s_types.Str):
         if len(parts) == 2:
             proto, valu = parts
 
-        if proto not in ('tcp', 'udp', 'icmp', 'host'):
-            raise s_exc.BadTypeValu(valu=orig, name=self.name,
-                                    mesg='inet:addr protocol must be in: tcp, udp, icmp, host')
+        if proto not in self.protos:
+            protostr = ','.join(self.protos)
+            mesg = f'inet:addr protocol must be one of: {protostr}'
+            raise s_exc.BadTypeValu(mesg=mesg, valu=orig, name=self.name)
+
         subs['proto'] = proto
 
         valu = valu.strip().strip('/')
@@ -181,6 +187,10 @@ class Addr(s_types.Str):
                     subs['port'] = port
                     portstr = f':{port}'
 
+                if port and proto in self.noports:
+                    mesg = f'Protocol {proto} does not allow specifying ports.'
+                    raise s_exc.BadTypeValu(mesg=mesg, valu=orig)
+
                 return f'{proto}://[{ipv6}]{portstr}', {'subs': subs}
 
             mesg = f'Invalid IPv6 w/port ({orig})'
@@ -196,6 +206,10 @@ class Addr(s_types.Str):
         if port:
             subs['port'] = port
 
+        if port and proto in self.noports:
+            mesg = f'Protocol {proto} does not allow specifying ports.'
+            raise s_exc.BadTypeValu(mesg=mesg, valu=orig)
+
         ipv4 = self.modl.type('inet:ipv4').norm(valu)[0]
         ipv4_repr = self.modl.type('inet:ipv4').repr(ipv4)
         subs['ipv4'] = ipv4

{synapse-2.216.0 → synapse-2.218.0}/synapse/models/orgs.py

@@ -292,11 +292,13 @@ class OuModule(s_module.CoreModule):
                     'doc': 'An organization enacting a document.'}),
 
                 ('ou:requirement:type:taxonomy', ('taxonomy', {}), {
+                    'deprecated': True,
                     'interfaces': ('meta:taxonomy',),
-                    'doc': 'A taxonomy of requirement types.'}),
+                    'doc': 'Deprecated. Please use doc:requirement and ou:enacted.'}),
 
                 ('ou:requirement', ('guid', {}), {
-                    'doc': 'A specific requirement.'}),
+                    'deprecated': True,
+                    'doc': 'Deprecated. Please use doc:requirement and ou:enacted.'}),
 
             ),
             'edges': (

{synapse-2.216.0 → synapse-2.218.0}/synapse/tests/test_lib_storm.py

@@ -4113,6 +4113,29 @@ class StormTest(s_t_utils.SynTest):
             view = await core.callStorm('return( $lib.view.get().iden )')
             fork = await core.callStorm('return( $lib.view.get().fork().iden )')
 
+            q = '''view.exec $view {
+                $lib.print(foo)
+                $lib.warn(bar)
+                [ it:dev:str=nomsg ]
+             }'''
+            msgs = await core.stormlist(q, opts={'view': fork, 'vars': {'view': view}})
+            self.stormIsInPrint('foo', msgs)
+            self.stormIsInWarn('bar', msgs)
+
+            q = '''
+                [ it:dev:str=woot ] $valu=$node.repr()
+                view.exec $view {
+                    $lib.print(foo)
+                    $lib.print($valu)
+                    $lib.warn(bar)
+                    [ it:dev:str=nomsg ]
+                }
+            '''
+            msgs = await core.stormlist(q, opts={'view': fork, 'vars': {'view': view}})
+            self.stormIsInPrint('foo', msgs)
+            self.stormIsInPrint('woot', msgs)
+            self.stormIsInWarn('bar', msgs)
+
             await core.addStormPkg({
                 'name': 'testpkg',
                 'version': (0, 0, 1),
@@ -4805,16 +4828,19 @@ class StormTest(s_t_utils.SynTest):
                 view1, layr1 = await core.callStorm('$view = $lib.view.get().fork() return(($view.iden, $view.layers.0.iden))')
                 view2, layr2 = await core.callStorm('$view = $lib.view.get().fork() return(($view.iden, $view.layers.0.iden))')
                 view3, layr3 = await core.callStorm('$view = $lib.view.get().fork() return(($view.iden, $view.layers.0.iden))')
+                view4, layr4 = await core.callStorm('$view = $lib.view.get().fork() return(($view.iden, $view.layers.0.iden))')
 
                 opts = {'vars': {
                     'view0': view0,
                     'view1': view1,
                     'view2': view2,
                     'view3': view3,
+                    'view4': view4,
                     'layr0': layr0,
                     'layr1': layr1,
                     'layr2': layr2,
                     'layr3': layr3,
+                    'layr4': layr4,
                 }}
 
                 # lets get some auth denies...
@@ -4867,13 +4893,26 @@ class StormTest(s_t_utils.SynTest):
                 self.len(3, await core.nodes('ps:contact', opts={'view': view1}))
                 self.len(3, await core.nodes('ps:contact', opts={'view': view2}))
 
-                # Check offset reporting
+                # Check offset reporting from pack()
                 q = '$layer=$lib.layer.get($layr0) return ($layer.pack())'
                 layrinfo = await core.callStorm(q, opts=opts)
                 pushs = layrinfo.get('pushs')
                 self.len(1, pushs)
                 pdef = list(pushs.values())[0]
-                self.lt(10, pdef.get('offs', 0))
+                eoffs = pdef.get('offs', 0)
+                self.lt(10, eoffs)
+
+                # check offset reporting from list()
+                msgs = await core.stormlist('layer.push.list $layr0', opts=opts)
+                self.stormIsInPrint(f'{eoffs}', msgs)
+
+                # Pull from layr0 using a custom offset (skip first node)
+                await core.callStorm(f'$lib.layer.get($layr0).addPush("tcp://root:secret@127.0.0.1:{port}/*/layer/{layr4}", offs={offs})', opts=opts)
+                await core.layers.get(layr4).waitEditOffs(offs, timeout=3)
+                self.len(2, await core.nodes('ps:contact', opts={'view': view4}))
+
+                # Clean up
+                self.none(await core.callStorm('$lib.layer.get($layr0).delPush($layr4)', opts=opts))
 
                 q = '$layer=$lib.layer.get($layr2) return ($layer.pack())'
                 layrinfo = await core.callStorm(q, opts=opts)
@@ -4914,7 +4953,7 @@ class StormTest(s_t_utils.SynTest):
                         }
                     }
                 ''')
-                self.eq(actv - 2, len(core.activecoros))
+                self.eq(actv - 3, len(core.activecoros))
                 tasks = await core.callStorm('return($lib.ps.list())')
                 self.len(0, [t for t in tasks if t.get('name').startswith('layer pull:')])
                 self.len(0, [t for t in tasks if t.get('name').startswith('layer push:')])
@@ -4984,7 +5023,7 @@ class StormTest(s_t_utils.SynTest):
                 tasks = await core.callStorm('return($lib.ps.list())')
                 self.len(1, [t for t in tasks if t.get('name').startswith('layer pull:')])
                 self.len(1, [t for t in tasks if t.get('name').startswith('layer push:')])
-                self.eq(actv, len(core.activecoros))
+                self.eq(actv - 1, len(core.activecoros))
 
                 pushpulls = set()
                 for ldef in await core.getLayerDefs():
@@ -5009,7 +5048,7 @@ class StormTest(s_t_utils.SynTest):
                 tasks = await core.callStorm('return($lib.ps.list())')
                 self.len(0, [t for t in tasks if t.get('name').startswith('layer pull:')])
                 self.len(0, [t for t in tasks if t.get('name').startswith('layer push:')])
-                self.eq(actv - 2, len(core.activecoros))
+                self.eq(actv - 3, len(core.activecoros))
 
                 with self.raises(s_exc.SchemaViolation):
                     await core.addLayrPush('newp', {})
@@ -5354,6 +5393,32 @@ class StormTest(s_t_utils.SynTest):
             for node in nodes:
                 self.none(node.tags.get('btag'))
 
+            q = '''runas visi {
+                $lib.print(foo)
+                $lib.warn(bar)
+                [ it:dev:str=nomsg ]
+             }'''
+            msgs = await core.stormlist(q)
+            self.stormIsInPrint('foo', msgs)
+            self.stormIsInWarn('bar', msgs)
+
+            q = '''
+                [it:dev:str=woot] $valu=$node.repr()
+                runas visi {
+                    $lib.print(foo)
+                    $lib.warn(bar)
+                    $lib.print($valu)
+                    [ it:dev:str=nomsg ]
+                }
+            '''
+            msgs = await core.stormlist(q)
+            self.stormIsInPrint('foo', msgs)
+            self.stormIsInPrint('woot', msgs)
+            self.stormIsInWarn('bar', msgs)
+
+            msgs = await core.stormlist('runas visi {$lib.raise(Foo, asdf)}')
+            self.stormIsInErr('asdf', msgs)
+
     async def test_storm_batch(self):
         async with self.getTestCore() as core:
             q = '''

{synapse-2.216.0 → synapse-2.218.0}/synapse/tests/test_model_inet.py

@@ -1363,6 +1363,27 @@ class InetModelTest(s_t_utils.SynTest):
                 for p, v in props.items():
                     self.eq(node.get(p), v)
 
+            nodes = await core.nodes('[ inet:server=gre://::1 ]')
+            self.eq(nodes[0].get('proto'), 'gre')
+
+            nodes = await core.nodes('[ inet:server=gre://1.2.3.4 ]')
+            self.eq(nodes[0].get('proto'), 'gre')
+
+            with self.raises(s_exc.BadTypeValu) as ctx:
+                await core.nodes('[ inet:server=gre://1.2.3.4:99 ]')
+
+            self.eq(ctx.exception.get('mesg'), 'Protocol gre does not allow specifying ports.')
+
+            with self.raises(s_exc.BadTypeValu) as ctx:
+                await core.nodes('[ inet:server="gre://[::1]:99" ]')
+
+            self.eq(ctx.exception.get('mesg'), 'Protocol gre does not allow specifying ports.')
+
+            with self.raises(s_exc.BadTypeValu) as ctx:
+                await core.nodes('[ inet:server=newp://1.2.3.4:99 ]')
+
+            self.eq(ctx.exception.get('mesg'), 'inet:addr protocol must be one of: tcp,udp,icmp,host,gre')
+
     async def test_servfile(self):
         async with self.getTestCore() as core:
             valu = ('tcp://127.0.0.1:4040', 64 * 'f')

{synapse-2.216.0 → synapse-2.218.0}/synapse/tests/test_model_infotech.py

@@ -1953,6 +1953,12 @@ class InfotechModelTest(s_t_utils.SynTest):
             self.len(1, nodes)
             self.eq(nodes[0].get('product'), 'product%23')
 
+    async def test_infotech_cpe_conversions(self):
+        self.thisEnvMust('CIRCLECI')
+
+        async with self.getTestCore() as core:
+            cpe23 = core.model.type('it:sec:cpe')
+            cpe22 = core.model.type('it:sec:cpe:v2_2')
             # Test 2.2->2.3 and 2.3->2.2 conversions
             filename = s_t_files.getAssetPath('cpedata.json')
             with open(filename, 'r') as fp:
@@ -1960,25 +1966,24 @@ class InfotechModelTest(s_t_utils.SynTest):
 
             for (_cpe22, _cpe23) in cpedata:
                 # Convert cpe22 -> cpe23
-                norm, info = cpe23.norm(_cpe22)
-                self.eq(norm, _cpe23)
+                norm_22, _ = cpe23.norm(_cpe22)
+                self.eq(norm_22, _cpe23)
 
-                norm, info = cpe23.norm(_cpe23)
-                self.eq(norm, _cpe23)
+                norm_23, info_23 = cpe23.norm(_cpe23)
+                self.eq(norm_23, _cpe23)
 
                 # No escaped characters in the secondary props
-                for name, valu in info.items():
+                for name, valu in info_23.items():
                     if name == 'v2_2':
                         continue
 
                     self.notin('\\', valu)
 
                 # Norm cpe23 and check the cpe22 conversion
-                norm, info = cpe23.norm(_cpe23)
-                v2_2 = info['subs']['v2_2']
+                sub_23_v2_2 = info_23['subs']['v2_2']
 
-                norm, info = cpe22.norm(v2_2)
-                self.eq(norm, _cpe22)
+                norm_sub_23_v2_2, _ = cpe22.norm(sub_23_v2_2)
+                self.eq(norm_sub_23_v2_2, sub_23_v2_2)
 
     async def test_cpe_scrape_one_to_one(self):
 

{synapse-2.216.0 → synapse-2.218.0}/synapse/tests/test_telepath.py

@@ -862,8 +862,7 @@ class TeleTest(s_t_utils.SynTest):
         self.isin('Cell path does not exist', cm.exception.get('mesg'))
 
     async def test_ipv6(self):
-        if s_common.envbool('CIRCLECI'):
-            self.skip('ipv6 listener is not supported in circleci')
+        self.thisEnvMustNot('CIRCLECI')  # ipv6 listener not supported in circleci
 
         foo = Foo()
 

{synapse-2.216.0 → synapse-2.218.0/synapse.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.216.0
+Version: 2.218.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0
@@ -22,7 +22,7 @@ Requires-Dist: pyOpenSSL<24.3.0,>=24.0.0
 Requires-Dist: cryptography<44.0.0,>=43.0.1
 Requires-Dist: msgpack<1.2.0,>=1.0.5
 Requires-Dist: xxhash<3.6.0,>=1.4.4
-Requires-Dist: lmdb<1.7.0,>=1.2.1
+Requires-Dist: lmdb<1.8.0,>=1.7.0
 Requires-Dist: tornado<7.0.0,>=6.2.0
 Requires-Dist: regex>=2022.9.11
 Requires-Dist: PyYAML<6.1.0,>=5.4

{synapse-2.216.0 → synapse-2.218.0}/synapse.egg-info/requires.txt

@@ -2,7 +2,7 @@ pyOpenSSL<24.3.0,>=24.0.0
 cryptography<44.0.0,>=43.0.1
 msgpack<1.2.0,>=1.0.5
 xxhash<3.6.0,>=1.4.4
-lmdb<1.7.0,>=1.2.1
+lmdb<1.8.0,>=1.7.0
 tornado<7.0.0,>=6.2.0
 regex>=2022.9.11
 PyYAML<6.1.0,>=5.4