synapse 2.215.0__tar.gz → 2.217.0__tar.gz

{synapse-2.215.0/synapse.egg-info → synapse-2.217.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.215.0
+Version: 2.217.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-2.215.0 → synapse-2.217.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.215.0'
+version = '2.217.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -99,11 +99,16 @@ include = ["synapse*"]
 [tool.ruff]
 include = [
     'synapse/*.py',
+    'scripts/*.py',
     'examples/*.py',
 ]
 target-version = "py311"
 
 [tool.ruff.lint]
 select = [
+    "E251", # Whitespace around kwargs; preview rule
     "F821",  # Check for undefined names - this is generally fatal
+    "W191", "W191", "W292", "W293", "W391",  # Whitespace checks; W391 is a preview rule.
+    "W605", # invalid escape sequence
 ]
+preview = true

{synapse-2.215.0 → synapse-2.217.0}/synapse/cortex.py

@@ -758,6 +758,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     confbase['mirror']['hidedocs'] = False  # type: ignore
     confbase['mirror']['hidecmdl'] = False  # type: ignore
 
+    confbase['safemode']['hidecmdl'] = False
+    confbase['safemode']['description'] = (
+        'Enable safe-mode which disables crons, triggers, dmons, storm '
+        'package onload handlers, view merge tasks, and storm pools.'
+    )
+
     confdefs = {
         'axon': {
             'description': 'A telepath URL for a remote axon.',
@@ -1572,9 +1578,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if self.isactive:
             await self._checkLayerModels()
 
-        self.addActiveCoro(self.agenda.runloop)
+        if not self.safemode:
+            self.addActiveCoro(self.agenda.runloop)
 
         await self._initStormDmons()
+
         await self._initStormSvcs()
 
         # share ourself via the cell dmon as "cortex"
@@ -1621,6 +1629,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
     async def initStormPool(self):
 
+        if self.safemode:
+            return
+
         try:
 
             byts = self.slab.get(b'storm:pool', db='cell:conf')
@@ -2961,6 +2972,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if onload is not None and self.isactive:
             async def _onload():
+                if self.safemode:
+                    await self.fire('core:pkg:onload:skipped', pkg=name, reason='safemode')
+                    return
+
                 await self.fire('core:pkg:onload:start', pkg=name)
                 try:
                     async for mesg in self.storm(onload):

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/cell.py

@@ -930,6 +930,12 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
     confdefs = {}  # type: ignore  # This should be a JSONSchema properties list for an object.
     confbase = {
+        'safemode': {
+            'default': False,
+            'description': 'Boot the service in safe-mode.',
+            'type': 'boolean',
+            'hidecmdl': True,
+        },
         'cell:guid': {
             'description': 'An optional hard-coded GUID to store as the permanent GUID for the service.',
             'type': 'string',
@@ -1200,6 +1206,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'tasks': 1,
         }
 
+        self.safemode = self.conf.req('safemode')
+        if self.safemode:
+            mesg = f'Booting {self.getCellType()} in safe-mode. Some functionality may be disabled.'
+            logger.warning(mesg)
+
         self.minfree = self.conf.get('limit:disk:free')
         if self.minfree is not None:
             self.minfree = self.minfree / 100
@@ -4816,6 +4827,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                 'iden': self.getCellIden(),
                 'paused': self.paused,
                 'active': self.isactive,
+                'safemode': self.safemode,
                 'started': self.startms,
                 'ready': self.nexsroot.ready.is_set(),
                 'commit': self.COMMIT,

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/cmd.py

@@ -3,15 +3,19 @@ import asyncio
 import argparse
 
 import synapse.exc as s_exc
-import synapse.common as s_common
 
 import synapse.lib.coro as s_coro
 import synapse.lib.output as s_output
 
 class Parser(argparse.ArgumentParser):
+    '''
+    argparse.ArgumentParser helper class.
 
+    - exit() is overriden to raise a SynErr ( ParserExit )
+    - _print_message prints to an outp object
+    - description formatter uses argparse.RawDescriptionHelpFormatter
+    '''
     def __init__(self, prog=None, outp=s_output.stdout, **kwargs):
-
         self.outp = outp
         self.exited = False
 

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/layer.py

@@ -3489,7 +3489,10 @@ class Layer(s_nexus.Pusher):
 
         valt = edit[1]
         valu, stortype = valt
-        if sode.get('valu') == valt:
+
+        isarray = stortype & STOR_FLAG_ARRAY
+
+        if not isarray and sode.get('valu') == valt:
             return ()
 
         abrv = self.setPropAbrv(form, None)
@@ -3500,7 +3503,7 @@ class Layer(s_nexus.Pusher):
         sode['valu'] = valt
         self.setSodeDirty(buid, sode, form)
 
-        if stortype & STOR_FLAG_ARRAY:
+        if isarray:
 
             for indx in self.getStorIndx(stortype, valu):
                 self.layrslab.put(abrv + indx, buid, db=self.byarray)
@@ -3587,6 +3590,8 @@ class Layer(s_nexus.Pusher):
         if prop[0] == '.':  # '.' to detect universal props (as quickly as possible)
             univabrv = self.setPropAbrv(None, prop)
 
+        isarray = stortype & STOR_FLAG_ARRAY
+
         if oldv is not None:
 
             # merge intervals and min times
@@ -3599,7 +3604,7 @@ class Layer(s_nexus.Pusher):
             elif stortype == STOR_TYPE_MAXTIME:
                 valu = max(valu, oldv)
 
-            if valu == oldv and stortype == oldt:
+            if not isarray and valu == oldv and stortype == oldt:
                 return ()
 
             if oldt & STOR_FLAG_ARRAY:
@@ -3638,7 +3643,7 @@ class Layer(s_nexus.Pusher):
         sode['props'][prop] = (valu, stortype)
         self.setSodeDirty(buid, sode, form)
 
-        if stortype & STOR_FLAG_ARRAY:
+        if isarray:
 
             realtype = stortype & 0x7fff
 
@@ -3828,7 +3833,7 @@ class Layer(s_nexus.Pusher):
             kvpairs.append((tp_abrv + indx, buid))
             kvpairs.append((ftp_abrv + indx, buid))
 
-        await self.layrslab.putmulti(kvpairs, db=self.bytagprop)
+        self.layrslab._putmulti(kvpairs, db=self.bytagprop)
 
         return (
             (EDIT_TAGPROP_SET, (tag, prop, valu, oldv, stortype), ()),

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/schemas.py

@@ -297,11 +297,18 @@ _authRulesSchema = {
         'type': 'array',
         'items': [
             {'type': 'boolean'},
-            {'type': 'array', 'items': {'type': 'string'}},
+            {
+                'type': 'array',
+                'items': {
+                    'type': 'string',
+                    'minLength': 1
+                },
+                'minItems': 1
+            },
         ],
         'minItems': 2,
         'maxItems': 2,
-    }
+    },
 }
 reqValidRules = s_config.getJsValidator(_authRulesSchema)
 

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/snap.py

@@ -384,12 +384,22 @@ class ProtoNode:
 
         if norminfo is None:
             try:
-                valu, norminfo = prop.type.norm(valu)
+                if (isinstance(valu, dict) and isinstance(prop.type, s_types.Guid)
+                    and (form := self.ctx.snap.core.model.form(prop.type.name)) is not None):
+
+                    norms, props = await self.ctx.snap._normGuidNodeDict(form, valu)
+                    valu = await self.ctx.snap._addGuidNodeByDict(form, norms, props)
+                    norminfo = {}
+                else:
+                    valu, norminfo = prop.type.norm(valu)
+
             except s_exc.BadTypeValu as e:
-                oldm = e.get('mesg')
-                e.update({'prop': prop.name,
-                          'form': prop.form.name,
-                          'mesg': f'Bad prop value {prop.full}={valu!r} : {oldm}'})
+                if 'prop' not in e.errinfo:
+                    oldm = e.get('mesg')
+                    e.update({'prop': prop.name,
+                              'form': prop.form.name,
+                              'mesg': f'Bad prop value {prop.full}={valu!r} : {oldm}'})
+
                 if self.ctx.snap.strict:
                     raise e
                 await self.ctx.snap.warn(e)

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/storm.py

@@ -1363,7 +1363,7 @@ class DmonManager(s_base.Base):
         '''
         Start all the dmons.
         '''
-        if self.enabled:
+        if self.enabled or self.core.safemode:
             return
         dmons = list(self.dmons.values())
         if not dmons:

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/stormtypes.py

@@ -3653,6 +3653,12 @@ class LibFeed(Lib):
                       {'name': 'data', 'type': 'prim', 'desc': 'Data to send to the ingest function.', },
                   ),
                   'returns': {'type': 'null', }}},
+        {'name': 'fromAxon', 'desc': 'Load a syn.nodes formatted export from axon.',
+         'type': {'type': 'function', '_funcname': '_fromAxon',
+                   'args': (
+                       {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 of the file stored in the axon.', },
+                  ),
+                  'returns': {'type': 'int', 'desc': 'The number of nodes loaded.', }}},
     )
     _storm_lib_path = ('feed',)
 

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/types.py

@@ -625,6 +625,9 @@ class Guid(Type):
         )
 
     def _normPyList(self, valu):
+        if not valu:
+            mesg = 'Guid list values cannot be empty.'
+            raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=mesg)
         return s_common.guid(valu), {}
 
     def _normPyStr(self, valu):

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 215, 0)
+version = (2, 217, 0)
 verstring = '.'.join([str(x) for x in version])
-commit = 'f4d48839f7e5f74bacdcc2d1b122d9ade6f1def7'
+commit = 'fa725ddc1b27901036592ea24d075fbe121349fa'

{synapse-2.215.0 → synapse-2.217.0}/synapse/lib/view.py

@@ -260,6 +260,9 @@ class View(s_nexus.Pusher):  # type: ignore
         if self.merging: # pragma: no cover
             return
 
+        if self.core.safemode:
+            return
+
         if not await self.isMergeReady():
             return
 
@@ -380,6 +383,9 @@ class View(s_nexus.Pusher):  # type: ignore
         if not await self.core.isCellActive():
             return
 
+        if self.core.safemode:
+            return
+
         self.mergetask = self.core.schedCoro(self.runViewMerge())
 
     async def finiMergeTask(self):
@@ -568,6 +574,9 @@ class View(s_nexus.Pusher):  # type: ignore
         if not await self.core.isCellActive():
             return
 
+        if self.core.safemode:
+            return
+
         self.trigtask = self.schedCoro(self._trigQueueLoop())
 
     async def finiTrigTask(self):
@@ -1577,7 +1586,7 @@ class View(s_nexus.Pusher):  # type: ignore
 
     async def runTagAdd(self, node, tag, valu):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         # Run any trigger handlers
@@ -1585,21 +1594,21 @@ class View(s_nexus.Pusher):  # type: ignore
 
     async def runTagDel(self, node, tag, valu):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runTagDel(node, tag)
 
     async def runNodeAdd(self, node):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runNodeAdd(node)
 
     async def runNodeDel(self, node):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runNodeDel(node)
@@ -1608,21 +1617,21 @@ class View(s_nexus.Pusher):  # type: ignore
         '''
         Handle when a prop set trigger event fired
         '''
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runPropSet(node, prop, oldv)
 
     async def runEdgeAdd(self, n1, edge, n2):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runEdgeAdd(n1, edge, n2)
 
     async def runEdgeDel(self, n1, edge, n2):
 
-        if self.core.migration:
+        if self.core.migration or self.core.safemode:
             return
 
         await self.triggers.runEdgeDel(n1, edge, n2)

{synapse-2.215.0 → synapse-2.217.0}/synapse/models/inet.py

@@ -108,6 +108,10 @@ def getAddrScope(ipv6):
 
 class Addr(s_types.Str):
 
+    protos = ('tcp', 'udp', 'icmp', 'host', 'gre')
+    # TODO: this should include icmp and host but requires a migration
+    noports = ('gre',)
+
     def postTypeInit(self):
         s_types.Str.postTypeInit(self)
         self.setNormFunc(str, self._normPyStr)
@@ -140,9 +144,11 @@ class Addr(s_types.Str):
         if len(parts) == 2:
             proto, valu = parts
 
-        if proto not in ('tcp', 'udp', 'icmp', 'host'):
-            raise s_exc.BadTypeValu(valu=orig, name=self.name,
-                                    mesg='inet:addr protocol must be in: tcp, udp, icmp, host')
+        if proto not in self.protos:
+            protostr = ','.join(self.protos)
+            mesg = f'inet:addr protocol must be one of: {protostr}'
+            raise s_exc.BadTypeValu(mesg=mesg, valu=orig, name=self.name)
+
         subs['proto'] = proto
 
         valu = valu.strip().strip('/')
@@ -181,6 +187,10 @@ class Addr(s_types.Str):
                     subs['port'] = port
                     portstr = f':{port}'
 
+                if port and proto in self.noports:
+                    mesg = f'Protocol {proto} does not allow specifying ports.'
+                    raise s_exc.BadTypeValu(mesg=mesg, valu=orig)
+
                 return f'{proto}://[{ipv6}]{portstr}', {'subs': subs}
 
             mesg = f'Invalid IPv6 w/port ({orig})'
@@ -196,6 +206,10 @@ class Addr(s_types.Str):
         if port:
             subs['port'] = port
 
+        if port and proto in self.noports:
+            mesg = f'Protocol {proto} does not allow specifying ports.'
+            raise s_exc.BadTypeValu(mesg=mesg, valu=orig)
+
         ipv4 = self.modl.type('inet:ipv4').norm(valu)[0]
         ipv4_repr = self.modl.type('inet:ipv4').repr(ipv4)
         subs['ipv4'] = ipv4
@@ -1542,7 +1556,7 @@ class InetModule(s_module.CoreModule):
                     ('inet:service:app', ('guid', {}), {
                         'interfaces': ('inet:service:object',),
                         'template': {'service:base': 'application'},
-                        'doc': 'A platform specific application.'}),
+                        'doc': 'An application which is part of a service architecture.'}),
 
                     ('inet:service:instance', ('guid', {}), {
                         'doc': 'An instance of the platform such as Slack or Discord instances.'}),
@@ -3769,6 +3783,12 @@ class InetModule(s_module.CoreModule):
                         ('desc', ('str', {}), {
                             'disp': {'hint': 'text'},
                             'doc': 'A description of the platform specific application.'}),
+
+                        ('provider', ('ou:org', {}), {
+                            'doc': 'The organization which provides the application.'}),
+
+                        ('provider:name', ('ou:name', {}), {
+                            'doc': 'The name of the organization which provides the application.'}),
                     )),
 
                     ('inet:service:account', {}, (

{synapse-2.215.0 → synapse-2.217.0}/synapse/models/orgs.py

@@ -292,11 +292,13 @@ class OuModule(s_module.CoreModule):
                     'doc': 'An organization enacting a document.'}),
 
                 ('ou:requirement:type:taxonomy', ('taxonomy', {}), {
+                    'deprecated': True,
                     'interfaces': ('meta:taxonomy',),
-                    'doc': 'A taxonomy of requirement types.'}),
+                    'doc': 'Deprecated. Please use doc:requirement and ou:enacted.'}),
 
                 ('ou:requirement', ('guid', {}), {
-                    'doc': 'A specific requirement.'}),
+                    'deprecated': True,
+                    'doc': 'Deprecated. Please use doc:requirement and ou:enacted.'}),
 
             ),
             'edges': (

{synapse-2.215.0 → synapse-2.217.0}/synapse/models/risk.py

@@ -220,6 +220,8 @@ class RiskModule(s_module.CoreModule):
                     'doc': 'The threat cluster targeted the target node.'}),
                 (('risk:threat', 'uses', None), {
                     'doc': 'The threat cluster uses the target node.'}),
+                (('risk:threat', 'uses', 'inet:service:app'), {
+                    'doc': 'The threat cluster uses the online application.'}),
                 (('risk:attack', 'targets', None), {
                     'doc': 'The attack targeted the target node.'}),
                 (('risk:attack', 'uses', None), {