synapse 2.195.0__py311-none-any.whl → 2.196.0__py311-none-any.whl

synapse/lib/stormtypes.py

@@ -2381,6 +2381,50 @@ class LibAxon(Lib):
                       {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to calculate hashes for.', },
                   ),
                   'returns': {'type': 'dict', 'desc': 'A dictionary of additional hashes.', }}},
+
+        {'name': 'read', 'desc': '''
+            Read bytes from a file stored in the Axon by its SHA256 hash.
+
+            Examples:
+                Read 100 bytes starting at offset 0::
+
+                    $byts = $lib.axon.read($sha256, size=100)
+
+                Read 50 bytes starting at offset 200::
+
+                    $byts = $lib.axon.read($sha256, offs=200, size=50)
+            ''',
+         'type': {'type': 'function', '_funcname': 'read',
+                  'args': (
+                      {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 hash of the file to read.'},
+                      {'name': 'offs', 'type': 'int', 'default': 0,
+                       'desc': 'The offset to start reading from.'},
+                      {'name': 'size', 'type': 'int', 'default': s_const.mebibyte,
+                       'desc': 'The number of bytes to read. Max is 1 MiB.'},
+                  ),
+                  'returns': {'type': 'bytes', 'desc': 'The requested bytes from the file.'}}},
+
+        {'name': 'unpack', 'desc': '''
+            Unpack bytes from a file stored in the Axon into a struct using the specified format.
+
+            Examples:
+                Unpack two 32-bit integers from the start of a file::
+
+                    $nums = $lib.axon.unpack($sha256, '<II')
+
+                Unpack a 64-bit float starting at offset 100::
+
+                    $float = $lib.axon.unpack($sha256, '<d', offs=100)
+            ''',
+         'type': {'type': 'function', '_funcname': 'unpack',
+                  'args': (
+                      {'name': 'sha256', 'type': 'str', 'desc': 'The SHA256 hash of the file to read.'},
+                      {'name': 'fmt', 'type': 'str', 'desc': 'The struct format string.'},
+                      {'name': 'offs', 'type': 'int', 'default': 0,
+                       'desc': 'The offset to start reading from.'},
+                  ),
+                  'returns': {'type': 'list', 'desc': 'The unpacked values as a tuple.'}}},
+
         {'name': 'upload', 'desc': '''
             Upload a stream of bytes to the Axon as a file.
 
@@ -2426,6 +2470,8 @@ class LibAxon(Lib):
             'size': self.size,
             'upload': self.upload,
             'hashset': self.hashset,
+            'read': self.read,
+            'unpack': self.unpack,
         }
 
     @stormfunc(readonly=True)
@@ -2712,6 +2758,48 @@ class LibAxon(Lib):
         await self.runt.snap.core.getAxon()
         return await self.runt.snap.core.axon.hashset(s_common.uhex(sha256))
 
+    @stormfunc(readonly=True)
+    async def read(self, sha256, offs=0, size=s_const.mebibyte):
+        '''
+        Read bytes from a file in the Axon.
+        '''
+        sha256 = await tostr(sha256)
+        size = await toint(size)
+        offs = await toint(offs)
+
+        if size > s_const.mebibyte:
+            mesg = f'Size must be between 1 and {s_const.mebibyte} bytes'
+            raise s_exc.BadArg(mesg=mesg)
+
+        if not self.runt.allowed(('axon', 'get')):
+            self.runt.confirm(('storm', 'lib', 'axon', 'get'))
+
+        await self.runt.snap.core.getAxon()
+
+        byts = b''
+        async for chunk in self.runt.snap.core.axon.get(s_common.uhex(sha256), offs=offs, size=size):
+            byts += chunk
+        return byts
+
+    @stormfunc(readonly=True)
+    async def unpack(self, sha256, fmt, offs=0):
+        '''
+        Unpack bytes from a file in the Axon using struct.
+        '''
+        if self.runt.snap.core.axoninfo.get('features', {}).get('unpack', 0) < 1:
+            mesg = 'The connected Axon does not support the the unpack API. Please update your Axon.'
+            raise s_exc.FeatureNotSupported(mesg=mesg)
+
+        sha256 = await tostr(sha256)
+        fmt = await tostr(fmt)
+        offs = await toint(offs)
+
+        if not self.runt.allowed(('axon', 'get')):
+            self.runt.confirm(('storm', 'lib', 'axon', 'get'))
+
+        await self.runt.snap.core.getAxon()
+        return await self.runt.snap.core.axon.unpack(s_common.uhex(sha256), fmt, offs)
+
 @registry.registerLib
 class LibBytes(Lib):
     '''
@@ -5446,7 +5534,7 @@ class Number(Prim):
         try:
             valu = s_common.hugenum(valu)
         except (TypeError, decimal.DecimalException) as e:
-            mesg = f'Failed to make number from {valu!r}'
+            mesg = f'Failed to make number from {s_common.trimText(repr(valu))}'
             raise s_exc.BadCast(mesg=mesg) from e
 
         Prim.__init__(self, valu, path=path)
@@ -6999,6 +7087,22 @@ class Layer(Prim):
                   'returns': {'name': 'Yields', 'type': 'node',
                               'desc': 'Yields nodes.', }}},
 
+        {'name': 'liftByNodeData', 'desc': '''
+            Lift and yield nodes with the given node data key set within the layer.
+
+            Example:
+                Yield all nodes with the data key zootsuit set in the top layer::
+
+                    yield $lib.layer.get().liftByNodeData(zootsuit)
+
+            ''',
+         'type': {'type': 'function', '_funcname': 'liftByNodeData',
+                  'args': (
+                      {'name': 'name', 'type': 'str', 'desc': 'The node data name to lift by.'},
+                  ),
+                  'returns': {'name': 'Yields', 'type': 'node',
+                              'desc': 'Yields nodes.', }}},
+
         {'name': 'getEdges', 'desc': '''
             Yield (n1iden, verb, n2iden) tuples for any light edges in the layer.
 
@@ -7110,6 +7214,7 @@ class Layer(Prim):
             'getEdges': self.getEdges,
             'liftByTag': self.liftByTag,
             'liftByProp': self.liftByProp,
+            'liftByNodeData': self.liftByNodeData,
             'getTagCount': self._methGetTagCount,
             'getPropCount': self._methGetPropCount,
             'getPropValues': self._methGetPropValues,
@@ -7177,6 +7282,19 @@ class Layer(Prim):
         async for _, buid, sode in layr.liftByPropValu(liftform, liftprop, cmprvals):
             yield await self.runt.snap._joinStorNode(buid, {iden: sode})
 
+    @stormfunc(readonly=True)
+    async def liftByNodeData(self, name):
+
+        name = await tostr(name)
+
+        iden = self.valu.get('iden')
+        layr = self.runt.snap.core.getLayer(iden)
+
+        await self.runt.reqUserCanReadLayer(iden)
+
+        async for _, buid, sode in layr.liftByDataName(name):
+            yield await self.runt.snap._joinStorNode(buid, {iden: sode})
+
     @stormfunc(readonly=True)
     async def getMirrorStatus(self):
         iden = self.valu.get('iden')
@@ -9813,7 +9931,7 @@ async def tostr(valu, noneok=False):
 
         return str(valu)
     except Exception as e:
-        mesg = f'Failed to make a string from {valu!r}.'
+        mesg = f'Failed to make a string from {s_common.trimText(repr(valu))}.'
         raise s_exc.BadCast(mesg=mesg) from e
 
 async def tobool(valu, noneok=False):
@@ -9827,7 +9945,7 @@ async def tobool(valu, noneok=False):
     try:
         return bool(valu)
     except Exception:
-        mesg = f'Failed to make a boolean from {valu!r}.'
+        mesg = f'Failed to make a boolean from {s_common.trimText(repr(valu))}.'
         raise s_exc.BadCast(mesg=mesg)
 
 async def tonumber(valu, noneok=False):
@@ -9852,13 +9970,13 @@ async def toint(valu, noneok=False):
         try:
             return int(valu, 0)
         except ValueError as e:
-            mesg = f'Failed to make an integer from {valu!r}.'
+            mesg = f'Failed to make an integer from {s_common.trimText(repr(valu))}.'
             raise s_exc.BadCast(mesg=mesg) from e
 
     try:
         return int(valu)
     except Exception as e:
-        mesg = f'Failed to make an integer from {valu!r}.'
+        mesg = f'Failed to make an integer from {s_common.trimText(repr(valu))}.'
         raise s_exc.BadCast(mesg=mesg) from e
 
 async def toiter(valu, noneok=False):
@@ -9877,7 +9995,7 @@ async def toiter(valu, noneok=False):
             async for item in agen:
                 yield item
     except TypeError as e:
-        mesg = f'Value is not iterable: {valu!r}'
+        mesg = f'Value is not iterable: {s_common.trimText(repr(valu))}'
         raise s_exc.StormRuntimeError(mesg=mesg) from e
 
 async def torepr(valu, usestr=False):

synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 195, 0)
+version = (2, 196, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'dc93864afdb2150e6d83ac4ed426c8e99bb3a448'
+commit = '8182360d2ebf8e8bdd00cb72f03812061c819cc2'

synapse/models/base.py

@@ -193,6 +193,9 @@ class BaseModule(s_module.CoreModule):
                     ('url', ('inet:url', {}), {
                         'doc': 'A URL which documents the meta source.'}),
 
+                    ('ingest:cursor', ('str', {}), {
+                        'doc': 'Used by ingest logic to capture the current ingest cursor within a feed.'}),
+
                     ('ingest:latest', ('time', {}), {
                         'doc': 'Used by ingest logic to capture the last time a feed ingest ran.'}),
 

synapse/models/infotech.py

@@ -1417,6 +1417,36 @@ class ItModule(s_module.CoreModule):
                         'deprecated': True,
                         'doc': 'Deprecated. Please use risk:vuln:cve:references.'}),
 
+                    ('nist:nvd:source', ('ou:name', {}), {
+                        'doc': 'The name of the organization which reported the vulnerability to NIST.'}),
+
+                    ('nist:nvd:published', ('time', {}), {
+                        'doc': 'The date the vulnerability was first published in the NVD.'}),
+
+                    ('nist:nvd:modified', ('time', {"ismax": True}), {
+                        'doc': 'The date the vulnerability was last modified in the NVD.'}),
+
+                    ('cisa:kev:name', ('str', {}), {
+                        'doc': 'The name of the vulnerability according to the CISA KEV database.'}),
+
+                    ('cisa:kev:desc', ('str', {}), {
+                        'doc': 'The description of the vulnerability according to the CISA KEV database.'}),
+
+                    ('cisa:kev:action', ('str', {}), {
+                        'doc': 'The action to mitigate the vulnerability according to the CISA KEV database.'}),
+
+                    ('cisa:kev:vendor', ('ou:name', {}), {
+                        'doc': 'The vendor name listed in the CISA KEV database.'}),
+
+                    ('cisa:kev:product', ('it:prod:softname', {}), {
+                        'doc': 'The product name listed in the CISA KEV database.'}),
+
+                    ('cisa:kev:added', ('time', {}), {
+                        'doc': 'The date the vulnerability was added to the CISA KEV database.'}),
+
+                    ('cisa:kev:duedate', ('time', {}), {
+                        'doc': 'The date the action is due according to the CISA KEV database.'}),
+
                 )),
                 ('it:sec:cpe', {}, (
                     ('v2_2', ('it:sec:cpe:v2_2', {}), {
@@ -2248,29 +2278,34 @@ class ItModule(s_module.CoreModule):
                         'doc': 'Software architecture.',
                     }),
                     ('released', ('time', {}), {
-                        'doc': 'Timestamp for when this version of the software was released.',
-                    }),
+                        'doc': 'Timestamp for when this version of the software was released.'}),
+
                     ('semver', ('it:semver', {}), {
-                        'doc': 'System normalized semantic version number.',
-                    }),
+                        'doc': 'System normalized semantic version number.'}),
+
                     ('semver:major', ('int', {}), {
-                        'doc': 'Version major number.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use semver range queries.'}),
+
                     ('semver:minor', ('int', {}), {
-                        'doc': 'Version minor number.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use semver range queries.'}),
+
                     ('semver:patch', ('int', {}), {
-                        'doc': 'Version patch number.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use semver range queries.'}),
+
                     ('semver:pre', ('str', {}), {
-                        'doc': 'Semver prerelease string.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated.'}),
+
                     ('semver:build', ('str', {}), {
-                        'doc': 'Semver build string.',
-                    }),
+                        'deprecated': True,
+                        'doc': 'Deprecated.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'URL where a specific version of the software is available from.',
-                    }),
+                        'doc': 'URL where a specific version of the software is available from.'}),
+
                 )),
 
                 ('it:prod:softlib', {}, (
@@ -2347,6 +2382,10 @@ class ItModule(s_module.CoreModule):
                     ('signame', ('it:av:signame', {}), {
                         'doc': 'The name of the signature returned by the scanner.'}),
 
+                    ('categories', ('array', {'sorted': True, 'uniq': True,
+                            'type': 'str', 'typeopts': {'lower': True, 'onespace': True}}), {
+                        'doc': 'A list of categories for the result returned by the scanner.'}),
+
                     ('target:file', ('file:bytes', {}), {
                         'doc': 'The file that was scanned to produce the result.'}),
 

synapse/models/orgs.py

@@ -1346,23 +1346,27 @@ class OuModule(s_module.CoreModule):
                     }),
                 )),
                 ('ou:contest:result', {}, (
+
                     ('contest', ('ou:contest', {}), {
                         'ro': True,
-                        'doc': 'The contest.',
-                    }),
+                        'doc': 'The contest that the participant took part in.'}),
+
                     ('participant', ('ps:contact', {}), {
                         'ro': True,
-                        'doc': 'The participant',
-                    }),
+                        'doc': 'The participant in the contest.'}),
+
                     ('rank', ('int', {}), {
-                        'doc': 'The rank order of the participant.',
-                    }),
+                        'doc': "The participant's rank order in the contest."}),
+
                     ('score', ('int', {}), {
-                        'doc': 'The score of the participant.',
-                    }),
+                        'doc': "The participant's final score in the contest."}),
+
+                    ('period', ('ival', {}), {
+                        'doc': 'The period of time when the participant competed in the contest.'}),
+
                     ('url', ('inet:url', {}), {
-                        'doc': 'The contest result website URL.',
-                    }),
+                        'doc': 'The contest result website URL.'}),
+
                 )),
                 ('ou:enacted:status:taxonomy', {}, ()),
                 ('ou:enacted', {}, (

synapse/models/risk.py

@@ -492,6 +492,9 @@ class RiskModule(s_module.CoreModule):
                     ('id', ('str', {'strip': True}), {
                         'doc': 'An identifier for the vulnerability.'}),
 
+                    ('tag', ('syn:tag', {}), {
+                        'doc': 'A tag used to annotate the presence or use of the vulnerability.'}),
+
                     ('cve', ('it:sec:cve', {}), {
                         'doc': 'The CVE ID of the vulnerability.'}),
 
@@ -506,34 +509,44 @@ class RiskModule(s_module.CoreModule):
                         'doc': 'An array of documentation URLs provided by the CVE database.'}),
 
                     ('nist:nvd:source', ('ou:name', {}), {
-                        'doc': 'The name of the organization which reported the vulnerability to NIST.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:nist:nvd:source.'}),
 
                     ('nist:nvd:published', ('time', {}), {
-                        'doc': 'The date the vulnerability was first published in the NVD.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:nist:nvd:published.'}),
 
                     ('nist:nvd:modified', ('time', {"ismax": True}), {
-                        'doc': 'The date the vulnerability was last modified in the NVD.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:nist:nvd:modified.'}),
 
                     ('cisa:kev:name', ('str', {}), {
-                        'doc': 'The name of the vulnerability according to the CISA KEV database.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:cisa:kev:name.'}),
 
                     ('cisa:kev:desc', ('str', {}), {
-                        'doc': 'The description of the vulnerability according to the CISA KEV database.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:cisa:kev:desc.'}),
 
                     ('cisa:kev:action', ('str', {}), {
-                        'doc': 'The action to mitigate the vulnerability according to the CISA KEV database.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:cisa:kev:action.'}),
 
                     ('cisa:kev:vendor', ('ou:name', {}), {
-                        'doc': 'The vendor name listed in the CISA KEV database.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:cisa:kev:vendor.'}),
 
                     ('cisa:kev:product', ('it:prod:softname', {}), {
-                        'doc': 'The product name listed in the CISA KEV database.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:cisa:kev:product.'}),
 
                     ('cisa:kev:added', ('time', {}), {
-                        'doc': 'The date the vulnerability was added to the CISA KEV database.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:cisa:kev:added.'}),
 
                     ('cisa:kev:duedate', ('time', {}), {
-                        'doc': 'The date the action is due according to the CISA KEV database.'}),
+                        'deprecated': True,
+                        'doc': 'Deprecated. Please use it:sec:cve:cisa:kev:duedate.'}),
 
                     ('cvss:v2', ('cvss:v2', {}), {
                         'doc': 'The CVSS v2 vector for the vulnerability.'}),

synapse/models/transport.py

@@ -454,21 +454,26 @@ class TransportModule(s_module.CoreModule):
                     ('type', ('transport:sea:vessel:type:taxonomy', {}), {
                         'doc': 'The type of vessel.'}),
 
-                    # TODO: convert this to an entity:name
-                    ('name', ('str', {'lower': True, 'onespace': True}), {
-                        'doc': 'The name of the vessel'}),
+                    ('name', ('entity:name', {}), {
+                        'doc': 'The name of the vessel.'}),
+
                     ('length', ('geo:dist', {}), {
                         'deprecated': True,
                         'doc': 'Deprecated. Please use :phys:length.'}),
+
                     ('beam', ('geo:dist', {}), {
                         'doc': 'The official overall vessel beam'}),
+
                     ('flag', ('iso:3166:cc', {}), {
                         'doc': 'The country the vessel is flagged to.'}),
+
                     ('mmsi', ('transport:sea:mmsi', {}), {
                         'doc': 'The Maritime Mobile Service Identifier assigned to the vessel.'}),
+
                     ('make', ('str', {'lower': True, 'strip': True}), {
                         'deprecated': True,
                         'doc': 'Deprecated. Please use :manufacturer:name.'}),
+
                     ('operator', ('ps:contact', {}), {
                         'doc': 'The contact information of the operator.'}),
                     # TODO tonnage / gross tonnage?

synapse/telepath.py

@@ -321,6 +321,9 @@ class Aware:
         '''
         return self
 
+    async def getTeleFeats(self):
+        return {}
+
     def onTeleShare(self, dmon, name):
         pass
 
@@ -599,6 +602,8 @@ class Proxy(s_base.Base):
         self.shares = {}
 
         self._ahainfo = {}
+        self._features = {}
+
         self.sharinfo = {}
         self.methinfo = {}
 
@@ -634,6 +639,12 @@ class Proxy(s_base.Base):
         self.onfini(fini)
         self.link.onfini(self.fini)
 
+    def _hasTeleFeat(self, name, vers=1):
+        return self._features.get(name, 0) >= vers
+
+    def _hasTeleMeth(self, name):
+        return self.methinfo.get(name) is not None
+
     def _getSynVers(self):
         '''
         Helper method to retrieve the remote Synapse version from Proxy.
@@ -906,6 +917,7 @@ class Proxy(s_base.Base):
 
         self.sess = self.synack[1].get('sess')
         self._ahainfo = self.synack[1].get('ahainfo', {})
+        self._features = self.synack[1].get('features', {})
         self.sharinfo = self.synack[1].get('sharinfo', {})
         self.methinfo = self.sharinfo.get('meths', {})
 

synapse/tests/test_axon.py

@@ -4,6 +4,7 @@ import csv
 import sys
 import base64
 import shutil
+import struct
 import asyncio
 import hashlib
 import logging
@@ -489,6 +490,28 @@ bar baz",vv
             with self.raises(s_exc.BadArg):
                 bytslist = [b async for b in axon.get(sha256, 0, size=0)]
 
+        # test unpack
+        intdata = struct.pack('>QQQ', 1, 2, 3)
+        size, sha256 = await axon.put(intdata)
+        self.eq((1,), await axon.unpack(sha256, '>Q'))
+        self.eq((2,), await axon.unpack(sha256, '>Q', offs=8))
+        self.eq((3,), await axon.unpack(sha256, '>Q', offs=16))
+        self.eq((2, 3), await axon.unpack(sha256, '>QQ', offs=8))
+
+        fmt = 'Q' * 150_000
+        with self.raises(s_exc.BadArg) as cm:
+            await axon.unpack(sha256, '>' + fmt)
+        self.isin('Struct format would read too much data', cm.exception.get('mesg'))
+
+        with self.raises(s_exc.BadArg):
+            await axon.unpack(sha256, 'not a valid format')
+
+        with self.raises(s_exc.BadArg):
+            await axon.unpack(sha256, 123)
+
+        with self.raises(s_exc.BadDataValu):
+            await axon.unpack(sha256, '>Q', offs=24)
+
     async def test_axon_base(self):
         async with self.getTestAxon() as axon:
             self.isin('axon', axon.dmon.shared)

synapse/tests/test_common.py

@@ -12,6 +12,34 @@ import synapse.tests.utils as s_t_utils
 logger = logging.getLogger(__name__)
 
 class CommonTest(s_t_utils.SynTest):
+
+    async def test_waitgenr(self):
+
+        async def genr():
+            yield 10
+            raise Exception('omg')
+
+        rets = [retn async for retn in s_common.waitgenr(genr(), 10)]
+
+        self.true(rets[0][0])
+        self.false(rets[1][0])
+
+        async def one():
+            yield 'item'
+
+        rets = [retn async for retn in s_common.waitgenr(one(), timeout=1.0)]
+        self.eq(rets, [(True, 'item')])
+
+        async def genr():
+            yield 1
+            await asyncio.sleep(60)
+            yield 2
+
+        rets = [retn async for retn in s_common.waitgenr(genr(), timeout=0.1)]
+        self.eq(rets[0], (True, 1))
+        self.false(rets[1][0])
+        self.eq(rets[1][1]['err'], 'TimeoutError')
+
     def test_tuplify(self):
         tv = ['node', [['test:str', 'test'],
                        {'tags': {

synapse/tests/test_datamodel.py

@@ -334,3 +334,11 @@ class DataModelTest(s_t_utils.SynTest):
             self.none(core.model.edge(('meta:rule', 'matches', None)))
 
             core.model.delEdge(('meta:rule', 'matches', None))
+
+    async def test_datamodel_locked_subs(self):
+
+        async with self.getTestCore() as core:
+            await core.setDeprLock('it:prod:softver:semver:major', True)
+            nodes = await core.nodes('[ it:prod:softver=* :semver=3.1.0 ]')
+            self.none(nodes[0].get('semver:major'))
+            self.eq(1, nodes[0].get('semver:minor'))