synapse 2.194.0__tar.gz → 2.195.1__tar.gz

{synapse-2.194.0/synapse.egg-info → synapse-2.195.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse
-Version: 2.194.0
+Version: 2.195.1
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0

{synapse-2.194.0 → synapse-2.195.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.194.0'
+version = '2.195.1'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.194.0 → synapse-2.195.1}/synapse/cortex.py

@@ -2860,6 +2860,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 cmdtext = cdef.get('storm')
                 await self.getStormQuery(cmdtext)
 
+            if cdef.get('forms') is not None:
+                name = cdef.get('name')
+                mesg = f"Storm command definition 'forms' key is deprecated and will be removed " \
+                       f"in 3.0.0 (command {name} in package {pkgname})"
+                logger.warning(mesg, extra=await self.getLogExtra(name=name, pkgname=pkgname))
+
         for gdef in pkgdef.get('graphs', ()):
             gdef['iden'] = s_common.guid((pkgname, gdef.get('name')))
             gdef['scope'] = 'power-up'

{synapse-2.194.0 → synapse-2.195.1}/synapse/datamodel.py

@@ -583,11 +583,11 @@ class Model:
         item = s_types.Array(self, 'array', info, {'type': 'int'})
         self.addBaseType(item)
 
-        info = {'doc': 'An digraph edge base type.'}
+        info = {'doc': 'An digraph edge base type.', 'deprecated': True}
         item = s_types.Edge(self, 'edge', info, {})
         self.addBaseType(item)
 
-        info = {'doc': 'An digraph edge base type with a unique time.'}
+        info = {'doc': 'An digraph edge base type with a unique time.', 'deprecated': True}
         item = s_types.TimeEdge(self, 'timeedge', info, {})
         self.addBaseType(item)
 
@@ -1122,7 +1122,7 @@ class Model:
                 # warn but do not blow up. there may be extended model elements
                 # with {}s which are not used for templates...
                 if item.find('{') != -1: # pragma: no cover
-                    logger.warning(f'Missing template specifier in: {item}')
+                    logger.warning(f'Missing template specifier in: {item} on {form.name}')
 
                 return item
 

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/auth.py

@@ -881,13 +881,14 @@ class Role(Ruler):
     set of rules can be applied to multiple users.
     '''
     def pack(self):
-        return {
+        ret = {
             'type': 'role',
             'iden': self.iden,
             'name': self.name,
             'rules': self.info.get('rules'),
             'authgates': self.authgates,
         }
+        return s_msgpack.deepcopy(ret)
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
         if nexs:
@@ -956,7 +957,7 @@ class User(Ruler):
                 _roles.append(role.pack())
             roles = _roles
 
-        return {
+        ret = {
             'type': 'user',
             'iden': self.iden,
             'name': self.name,
@@ -968,6 +969,7 @@ class User(Ruler):
             'archived': self.info.get('archived'),
             'authgates': self.authgates,
         }
+        return s_msgpack.deepcopy(ret)
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
         if nexs:

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/link.py

@@ -15,7 +15,8 @@ import synapse.lib.base as s_base
 import synapse.lib.const as s_const
 import synapse.lib.msgpack as s_msgpack
 
-readsize = 10 * s_const.megabyte
+READSIZE = 16 * s_const.mebibyte
+MAXWRITE = 64 * s_const.mebibyte
 
 async def connect(host, port, ssl=None, hostname=None, linkinfo=None):
     '''
@@ -114,8 +115,8 @@ class Link(s_base.Base):
         self.sock = self.writer.get_extra_info('socket')
         self.peercert = self.writer.get_extra_info('peercert')
 
+        self._txlock = asyncio.Lock()
         self._forceclose = forceclose
-        self._drain_lock = asyncio.Lock()
 
         if info is None:
             info = {}
@@ -234,11 +235,18 @@ class Link(s_base.Base):
         return dict(self._addrinfo)
 
     async def send(self, byts):
-        self.writer.write(byts)
-        # Avoid Python bug.  See https://github.com/python/cpython/issues/74116
-        # TODO Remove drain lock in 3.10+
-        async with self._drain_lock:
-            await self.writer.drain()
+
+        offs = 0
+        size = len(byts)
+
+        async with self._txlock:
+
+            while offs < size:
+
+                self.writer.write(byts[offs:offs + MAXWRITE])
+                offs += MAXWRITE
+
+                await self.writer.drain()
 
     async def tx(self, mesg):
         '''
@@ -247,23 +255,29 @@ class Link(s_base.Base):
         if self.isfini:
             raise s_exc.IsFini()
 
+        offs = 0
         byts = s_msgpack.en(mesg)
-        try:
+        size = len(byts)
 
-            self.writer.write(byts)
-            # Avoid Python bug.  See https://github.com/python/cpython/issues/74116
-            # TODO Remove drain lock in 3.10+
-            async with self._drain_lock:
-                await self.writer.drain()
+        async with self._txlock:
 
-        except (asyncio.CancelledError, Exception) as e:
+            try:
+
+                while offs < size:
 
-            await self.fini()
+                    self.writer.write(byts[offs:offs + MAXWRITE])
+                    offs += MAXWRITE
 
-            einfo = s_common.retnexc(e)
-            logger.debug('link.tx connection trouble %s', einfo)
+                    await self.writer.drain()
 
-            raise
+            except (asyncio.CancelledError, Exception) as e:
+
+                await self.fini()
+
+                einfo = s_common.retnexc(e)
+                logger.debug('link.tx connection trouble %s', einfo)
+
+                raise
 
     def txfini(self):
         self.sock.shutdown(1)
@@ -294,7 +308,7 @@ class Link(s_base.Base):
 
             try:
 
-                byts = await self.reader.read(readsize)
+                byts = await self.reader.read(READSIZE)
                 if not byts:
                     await self.fini()
                     return None

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/modelrev.py

@@ -13,7 +13,7 @@ import synapse.models.infotech as s_infotech
 
 logger = logging.getLogger(__name__)
 
-maxvers = (0, 2, 31)
+maxvers = (0, 2, 32)
 
 class ModelRev:
 
@@ -50,6 +50,7 @@ class ModelRev:
             ((0, 2, 29), self.revModel_0_2_29),
             ((0, 2, 30), self.revModel_0_2_30),
             ((0, 2, 31), self.revModel_0_2_31),
+            ((0, 2, 32), self.revModel_0_2_32),
         )
 
     async def _uniqSortArray(self, todoprops, layers):
@@ -815,6 +816,10 @@ class ModelRev:
         await migr.revModel_0_2_31()
         await self._normFormSubs(layers, 'it:sec:cpe')
 
+    async def revModel_0_2_32(self, layers):
+        await self._normPropValu(layers, 'transport:air:craft:model')
+        await self._normPropValu(layers, 'transport:sea:vessel:model')
+
     async def runStorm(self, text, opts=None):
         '''
         Run storm code in a schedcoro and log the output messages.

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/nexus.py

@@ -341,13 +341,17 @@ class NexsRoot(s_base.Base):
         if meta is None:
             meta = {}
 
-        if (nexus := self._nexskids.get(nexsiden)) is None:
-            raise s_exc.NoSuchIden(mesg=f'No Nexus Pusher with iden {nexsiden}.', iden=nexsiden)
+        async with self.cell.nexslock:
+            if (nexus := self._nexskids.get(nexsiden)) is None:
+                mesg = f'No Nexus Pusher with iden {nexsiden} {event=} args={s_common.trimText(repr(args))} ' \
+                       f'kwargs={s_common.trimText(repr(kwargs))}'
+                raise s_exc.NoSuchIden(mesg=mesg, iden=nexsiden, event=event)
 
-        if event not in nexus._nexshands:
-            raise s_exc.NoSuchName(mesg=f'No Nexus handler for event {event}.', name=event)
+            if event not in nexus._nexshands:
+                mesg = f'No event handler for event {event} args={s_common.trimText(repr(args))} ' \
+                       f'kwargs={s_common.trimText(repr(kwargs))}'
+                raise s_exc.NoSuchName(mesg=mesg, iden=nexsiden, event=event)
 
-        async with self.cell.nexslock:
             self.reqNotReadOnly()
             # Keep a reference to the shielded task to ensure it isn't GC'd
             self.applytask = asyncio.create_task(self._eat((nexsiden, event, args, kwargs, meta)))

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/scrape.py

@@ -128,6 +128,8 @@ _cpe23_regex = r'''(?P<valu>cpe:2\.3:[aho\*-]
 (?::(([?]+|\*)?([a-z0-9-._]|\\[\\?*!"#$%&\'()+,/:;<=>@\[\]^`{|}~])+([?]+|\*)?|[*-])){4})
 '''
 
+path_parts_limit = 1024
+
 linux_path_regex = r'''
 (?<![\w\d]+)
 (?P<valu>
@@ -142,17 +144,23 @@ linux_path_rootdirs = (
     'sys', 'tmp', 'usr', 'var'
 )
 
+# https://docs.kernel.org/filesystems/path-lookup.html#the-symlink-stack
+linux_path_limit = 4096
+
 def linux_path_check(match: regex.Match):
     mnfo = match.groupdict()
     valu = mnfo.get('valu')
 
+    if len(valu) > linux_path_limit:
+        return None, {}
+
     path = pathlib.PurePosixPath(valu)
     parts = path.parts
 
     if parts[0] != '/':
         return None, {}
 
-    if len(parts) < 2:
+    if len(parts) < 2 or len(parts) > path_parts_limit:
         return None, {}
 
     if parts[1] not in linux_path_rootdirs:
@@ -177,17 +185,26 @@ windows_path_reserved = (
 
 windows_drive_paths = [f'{letter}:\\' for letter in string.ascii_lowercase]
 
+# https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
+windows_path_limit = 32_767
+
 # https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
 def windows_path_check(match: regex.Match):
     mnfo = match.groupdict()
     valu = mnfo.get('valu')
 
+    if len(valu) > windows_path_limit:
+        return None, {}
+
     path = pathlib.PureWindowsPath(valu)
     parts = path.parts
 
     if parts[0].lower() not in windows_drive_paths:
         return None, {}
 
+    if len(parts) > path_parts_limit:
+        return None, {}
+
     for part in parts:
         if part in windows_path_reserved:
             return None, {}

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/stormlib/auth.py

@@ -583,12 +583,26 @@ stormcmds = (
     {
         'name': 'auth.perms.list',
         'descr': 'Display a list of the current permissions defined within the Cortex.',
-        'cmdargs': (),
+        'cmdargs': (
+            ('--find', {'type': 'str', 'help': 'A search string for permissions.'}),
+        ),
         'storm': '''
 
             for $pdef in $lib.auth.getPermDefs() {
                 $perm = $lib.str.join(".", $pdef.perm)
 
+                if $cmdopts.find {
+                    $find = $cmdopts.find.lower()
+                    $match = (
+                        $perm.lower().find($find) != (null) or
+                        $pdef.desc.lower().find($find) != (null) or
+                        $pdef.gate.lower().find($find) != (null) or
+                        ($pdef.ex and $pdef.ex.lower().find($find) != (null))
+                    )
+
+                    if (not $match) { continue }
+                }
+
                 $lib.print($perm)
                 $lib.print(`    {$pdef.desc}`)
                 $lib.print(`    gate: {$pdef.gate}`)

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/stormlib/cell.py

@@ -120,6 +120,9 @@ class CellLib(s_stormtypes.Lib):
     A Storm Library for interacting with the Cortex.
     '''
     _storm_locals = (
+        {'name': 'iden', 'desc': 'The Cortex service identifier.',
+         'type': {'type': 'gtor', '_gtorfunc': '_getCellIden',
+                  'returns': {'type': 'str', 'desc': 'The Cortex service identifier.'}}},
         {'name': 'getCellInfo', 'desc': 'Return metadata specific for the Cortex.',
          'type': {'type': 'function', '_funcname': '_getCellInfo', 'args': (),
                   'returns': {'type': 'dict', 'desc': 'A dictionary containing metadata.', }}},
@@ -174,6 +177,10 @@ class CellLib(s_stormtypes.Lib):
     )
     _storm_lib_path = ('cell',)
 
+    def __init__(self, runt, name=()):
+        s_stormtypes.Lib.__init__(self, runt, name=name)
+        self.gtors['iden'] = self._getCellIden
+
     def getObjLocals(self):
         return {
             'getCellInfo': self._getCellInfo,
@@ -187,6 +194,10 @@ class CellLib(s_stormtypes.Lib):
             'uptime': self._uptime,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _getCellIden(self):
+        return self.runt.snap.core.getCellIden()
+
     async def _hotFixesApply(self):
         if not self.runt.isAdmin():
             mesg = '$lib.cell.stormFixesApply() requires admin privs.'

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/stormlib/infosec.py

@@ -515,6 +515,7 @@ class CvssLib(s_stormtypes.Lib):
     '''
     _storm_locals = (
         {'name': 'calculate', 'desc': 'Calculate the CVSS score values for an input risk:vuln node.',
+         'deprecated': {'eolvers': 'v3.0.0'},
          'type': {'type': 'function', '_funcname': 'calculate',
                   'args': (
                       {'name': 'node', 'type': 'node',
@@ -527,6 +528,7 @@ class CvssLib(s_stormtypes.Lib):
                   'returns': {'type': 'dict', 'desc': 'A dictionary containing the computed score and subscores.', }
         }},
         {'name': 'calculateFromProps', 'desc': 'Calculate the CVSS score values from a props dict.',
+         'deprecated': {'eolvers': 'v3.0.0'},
          'type': {'type': 'function', '_funcname': 'calculateFromProps',
                   'args': (
                       {'name': 'props', 'type': 'dict',

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/stormtypes.py

@@ -5446,7 +5446,7 @@ class Number(Prim):
         try:
             valu = s_common.hugenum(valu)
         except (TypeError, decimal.DecimalException) as e:
-            mesg = f'Failed to make number from {valu!r}'
+            mesg = f'Failed to make number from {s_common.trimText(repr(valu))}'
             raise s_exc.BadCast(mesg=mesg) from e
 
         Prim.__init__(self, valu, path=path)
@@ -9813,7 +9813,7 @@ async def tostr(valu, noneok=False):
 
         return str(valu)
     except Exception as e:
-        mesg = f'Failed to make a string from {valu!r}.'
+        mesg = f'Failed to make a string from {s_common.trimText(repr(valu))}.'
         raise s_exc.BadCast(mesg=mesg) from e
 
 async def tobool(valu, noneok=False):
@@ -9827,7 +9827,7 @@ async def tobool(valu, noneok=False):
     try:
         return bool(valu)
     except Exception:
-        mesg = f'Failed to make a boolean from {valu!r}.'
+        mesg = f'Failed to make a boolean from {s_common.trimText(repr(valu))}.'
         raise s_exc.BadCast(mesg=mesg)
 
 async def tonumber(valu, noneok=False):
@@ -9852,13 +9852,13 @@ async def toint(valu, noneok=False):
         try:
             return int(valu, 0)
         except ValueError as e:
-            mesg = f'Failed to make an integer from {valu!r}.'
+            mesg = f'Failed to make an integer from {s_common.trimText(repr(valu))}.'
             raise s_exc.BadCast(mesg=mesg) from e
 
     try:
         return int(valu)
     except Exception as e:
-        mesg = f'Failed to make an integer from {valu!r}.'
+        mesg = f'Failed to make an integer from {s_common.trimText(repr(valu))}.'
         raise s_exc.BadCast(mesg=mesg) from e
 
 async def toiter(valu, noneok=False):
@@ -9877,7 +9877,7 @@ async def toiter(valu, noneok=False):
             async for item in agen:
                 yield item
     except TypeError as e:
-        mesg = f'Value is not iterable: {valu!r}'
+        mesg = f'Value is not iterable: {s_common.trimText(repr(valu))}'
         raise s_exc.StormRuntimeError(mesg=mesg) from e
 
 async def torepr(valu, usestr=False):

{synapse-2.194.0 → synapse-2.195.1}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 194, 0)
+version = (2, 195, 1)
 verstring = '.'.join([str(x) for x in version])
-commit = 'fb479d376805f963f11c5029310325c13ef2c883'
+commit = '83cc00aa05086cbc4dfec2f820db8f392c76a012'

{synapse-2.194.0 → synapse-2.195.1}/synapse/models/geospace.py

@@ -390,15 +390,49 @@ class GeoModule(s_module.CoreModule):
                     }),
                 ),
 
+                'interfaces': (
+                    ('geo:locatable', {
+                        'doc': 'Properties common to items and events which may be geolocated.',
+                        'template': {'geo:locatable': 'item'},
+                        'props': (
+                            ('place', ('geo:place', {}), {
+                                'doc': 'The place where the {geo:locatable} was located.'}),
+
+                            ('place:loc', ('loc', {}), {
+                                'doc': 'The geopolitical location of the {geo:locatable}.'}),
+
+                            ('place:name', ('geo:name', {}), {
+                                'doc': 'The name of the place where the {geo:locatable} was located.'}),
+
+                            ('place:address', ('geo:address', {}), {
+                                'doc': 'The postal address of the place where the {geo:locatable} was located.'}),
+
+                            ('place:latlong', ('geo:latlong', {}), {
+                                'doc': 'The latlong where the {geo:locatable} was located.'}),
+
+                            ('place:latlong:accuracy', ('geo:dist', {}), {
+                                'doc': 'The accuracy of the latlong where the {geo:locatable} was located.'}),
+
+                            ('place:country', ('pol:country', {}), {
+                                'doc': 'The country where the {geo:locatable} was located.'}),
+
+                            ('place:country:code', ('pol:iso2', {}), {
+                                'doc': 'The country code where the {geo:locatable} was located.'}),
+                        ),
+                    }),
+                ),
+
                 'types': (
 
                     ('geo:nloc', ('comp', {'fields': (('ndef', 'ndef'), ('latlong', 'geo:latlong'), ('time', 'time'))}), {
                         'deprecated': True,
                         'doc': 'Records a node latitude/longitude in space-time.'
                     }),
+
                     ('geo:telem', ('guid', {}), {
-                        'doc': 'A geospatial position of a node at a given time. The node should be linked via -(seenat)> edges.',
-                    }),
+                        'interfaces': ('phys:object', 'geo:locatable'),
+                        'template': {'phys:object': 'object', 'geo:locatable': 'object'},
+                        'doc': 'The geospatial position and physical characteristics of a node at a given time.'}),
 
                     ('geo:json', ('data', {'schema': geojsonschema}), {
                         'doc': 'GeoJSON structured JSON data.'}),
@@ -441,9 +475,11 @@ class GeoModule(s_module.CoreModule):
                 ),
 
                 'edges': (
+
                     ((None, 'seenat', 'geo:telem'), {
                         'deprecated': True,
-                        'doc': 'Deprecated. Please use ``geo:telem:node``.'}),
+                        'doc': 'Deprecated. Please use geo:telem:node.'}),
+
                     (('geo:place', 'contains', 'geo:place'), {
                         'doc': 'The source place completely contains the target place.'}),
                 ),
@@ -475,18 +511,21 @@ class GeoModule(s_module.CoreModule):
                     )),
 
                     ('geo:telem', {}, (
+
                         ('time', ('time', {}), {
-                            'doc': 'The time that the node was at the position.'}),
+                            'doc': 'The time that the telemetry measurements were taken.'}),
+
                         ('desc', ('str', {}), {
                             'doc': 'A description of the telemetry sample.'}),
+
                         ('latlong', ('geo:latlong', {}), {
-                            'doc': 'The latitude/longitude reading at the time.'}),
+                            'deprecated': True,
+                            'doc': 'Deprecated. Please use :place:latlong.'}),
+
                         ('accuracy', ('geo:dist', {}), {
-                            'doc': 'The reported accuracy of the latlong telemetry reading.'}),
-                        ('place', ('geo:place', {}), {
-                            'doc': 'The place which includes the latlong value.'}),
-                        ('place:name', ('geo:name', {}), {
-                            'doc': 'The purported place name. Used for entity resolution.'}),
+                            'deprecated': True,
+                            'doc': 'Deprecated. Please use :place:latlong:accuracy.'}),
+
                         ('node', ('ndef', {}), {
                             'doc': 'The node that was observed at the associated time and place.'}),
                     )),
@@ -494,6 +533,9 @@ class GeoModule(s_module.CoreModule):
                     ('geo:place:taxonomy', {}, ()),
                     ('geo:place', {}, (
 
+                        ('id', ('str', {'strip': True}), {
+                            'doc': 'A type specific identifier such as an airport ID.'}),
+
                         ('name', ('geo:name', {}), {
                             'alts': ('names',),
                             'doc': 'The name of the place.'}),

{synapse-2.194.0 → synapse-2.195.1}/synapse/models/infotech.py

@@ -652,10 +652,9 @@ class ItModule(s_module.CoreModule):
                 ('it:hostname', ('str', {'strip': True, 'lower': True}), {
                     'doc': 'The name of a host or system.'}),
 
-
                 ('it:host', ('guid', {}), {
-                    'interfaces': ('inet:service:object',),
-                    'template': {'service:base': 'host'},
+                    'interfaces': ('inet:service:object', 'phys:object'),
+                    'template': {'service:base': 'host', 'phys:object': 'physical host'},
                     'doc': 'A GUID that represents a host or system.'}),
 
                 ('it:log:event:type:taxonomy', ('taxonomy', {}), {