synapse 2.172.0__tar.gz → 2.174.0__tar.gz

{synapse-2.172.0/synapse.egg-info → synapse-2.174.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: synapse
-Version: 2.172.0
+Version: 2.174.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0

{synapse-2.172.0 → synapse-2.174.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.172.0'
+version = '2.174.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.172.0 → synapse-2.174.0}/synapse/axon.py

@@ -493,7 +493,7 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
 
     async def wants(self, sha256s):
         '''
-        Get a list of sha256 values the axon does not have from a input list.
+        Get a list of sha256 values the axon does not have from an input list.
 
         Args:
             sha256s (list): A list of sha256 values as bytes.

{synapse-2.172.0 → synapse-2.174.0}/synapse/common.py

@@ -11,6 +11,7 @@ import types
 import base64
 import shutil
 import struct
+import typing
 import asyncio
 import decimal
 import fnmatch
@@ -379,7 +380,7 @@ def reqbytes(*paths):
     with reqfile(*paths) as fd:
         return fd.read()
 
-def genfile(*paths):
+def genfile(*paths) -> typing.BinaryIO:
     '''
     Create or open (for read/write) a file path join.
 
@@ -396,7 +397,7 @@ def genfile(*paths):
         to append.
 
     Returns:
-        io.BufferedRandom: A file-object which can be read/written too.
+        A file-object which can be read/written too.
     '''
     path = genpath(*paths)
     gendir(os.path.dirname(path))
@@ -535,13 +536,26 @@ def yamlload(*paths):
     with io.open(path, 'rb') as fd:
         return yamlloads(fd)
 
+def yamldump(obj, stream: typing.Optional[typing.BinaryIO] =None) -> bytes:
+    '''
+    Dump a object to yaml.
+
+    Args:
+        obj: The object to serialize.
+        stream: The optional stream to write the stream too.
+
+    Returns:
+        The raw yaml bytes if stream is not provided.
+    '''
+    return yaml.dump(obj, allow_unicode=True, default_flow_style=False,
+                     default_style='', explicit_start=True, explicit_end=True,
+                     encoding='utf8', stream=stream, Dumper=Dumper)
+
 def yamlsave(obj, *paths):
     path = genpath(*paths)
     with genfile(path) as fd:
         fd.truncate(0)
-        yaml.dump(obj, allow_unicode=True, default_flow_style=False,
-                  default_style='', explicit_start=True, explicit_end=True,
-                  encoding='utf8', stream=fd, Dumper=Dumper)
+        yamldump(obj, stream=fd)
 
 def yamlmod(obj, *paths):
     '''

{synapse-2.172.0 → synapse-2.174.0}/synapse/cortex.py

@@ -42,6 +42,7 @@ import synapse.lib.schemas as s_schemas
 import synapse.lib.spooled as s_spooled
 import synapse.lib.version as s_version
 import synapse.lib.urlhelp as s_urlhelp
+import synapse.lib.hashitem as s_hashitem
 import synapse.lib.jsonstor as s_jsonstor
 import synapse.lib.modelrev as s_modelrev
 import synapse.lib.stormsvc as s_stormsvc
@@ -949,6 +950,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         # Initialize our storage and views
         await self._initCoreAxon()
+        await self._initJsonStor()
 
         await self._initCoreLayers()
         await self._initCoreViews()
@@ -1009,8 +1011,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         await self._bumpCellVers('cortex:storage', (
             (1, self._storUpdateMacros),
             (2, self._storLayrFeedDefaults),
+            (3, self._updateTriggerViewIdens),
         ), nexs=False)
 
+    async def _updateTriggerViewIdens(self):
+        for view in self.views.values():
+            for trigiden, trigger in await view.listTriggers():
+                if trigger.get('view') != view.iden:
+                    trigger.tdef['view'] = view.iden
+                    await view.trigdict.set(trigiden, trigger.tdef)
+
     async def _viewNomergeToProtected(self):
         for view in self.views.values():
             nomerge = view.info.get('nomerge', False)
@@ -1462,8 +1472,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     async def initServiceRuntime(self):
 
         # do any post-nexus initialization here...
-        await self._initJsonStor()
-
         if self.isactive:
             await self._checkNexsIndx()
 
@@ -2550,7 +2558,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         name = pkgdef.get('name')
         olddef = self.pkghive.get(name, None)
         if olddef is not None:
-            await self._dropStormPkg(olddef)
+            if s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
+                await self._dropStormPkg(olddef)
+            else:
+                return
 
         await self.loadStormPkg(pkgdef)
         await self.pkghive.set(name, pkgdef)
@@ -2965,17 +2976,19 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         '''
         Delete storm packages associated with a service.
         '''
-        oldpkgs = []
-        for _, pdef in self.pkghive.items():
-            pkgiden = pdef.get('svciden')
-            if pkgiden and pkgiden == iden:
-                oldpkgs.append(pdef)
-
-        for pkg in oldpkgs:
+        for pkg in self.getStormSvcPkgs(iden):
             name = pkg.get('name')
             if name:
                 await self._delStormPkg(name)
 
+    def getStormSvcPkgs(self, iden):
+        pkgs = []
+        for _, pdef in self.pkghive.items():
+            pkgiden = pdef.get('svciden')
+            if pkgiden and pkgiden == iden:
+                pkgs.append(pdef)
+        return pkgs
+
     async def setStormSvcEvents(self, iden, edef):
         '''
         Set the event callbacks for a storm service. Extends the sdef dict.
@@ -6113,6 +6126,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         Args:
             iden (bytes):  The iden of the cron job to be deleted
         '''
+        await self._killCronTask(iden)
         try:
             await self.agenda.delete(iden)
         except s_exc.NoSuchIden:
@@ -6142,6 +6156,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         '''
         await self.agenda.enable(iden)
         await self.feedBeholder('cron:enable', {'iden': iden}, gates=[iden])
+        logger.info(f'Enabled cron job {iden}', extra=await self.getLogExtra(iden=iden, status='MODIFY'))
 
     @s_nexus.Pusher.onPushAuto('cron:disable')
     async def disableCronJob(self, iden):
@@ -6152,7 +6167,28 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             iden (bytes):  The iden of the cron job to be changed
         '''
         await self.agenda.disable(iden)
+        await self._killCronTask(iden)
         await self.feedBeholder('cron:disable', {'iden': iden}, gates=[iden])
+        logger.info(f'Disabled cron job {iden}', extra=await self.getLogExtra(iden=iden, status='MODIFY'))
+
+    async def killCronTask(self, iden):
+        if self.agenda.appts.get(iden) is None:
+            return False
+        return await self._push('cron:task:kill', iden)
+
+    @s_nexus.Pusher.onPush('cron:task:kill')
+    async def _killCronTask(self, iden):
+
+        appt = self.agenda.appts.get(iden)
+        if appt is None:
+            return False
+
+        task = appt.task
+        if task is None:
+            return False
+
+        self.schedCoro(task.kill())
+        return True
 
     async def listCronJobs(self):
         '''

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/agenda.py

@@ -274,6 +274,7 @@ class _Appt:
     def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False):
         self.doc = ''
         self.name = ''
+        self.task = None
         self.stor = stor
         self.pool = pool
         self.iden = iden
@@ -801,7 +802,12 @@ class Agenda(s_base.Base):
 
         coro = self._runJob(user, appt)
         task = self.core.runActiveTask(coro)
+
         appt.task = await self.core.boss.promotetask(task, f'Cron {appt.iden}', user, info=info)
+        async def fini():
+            appt.task = None
+
+        appt.task.onfini(fini)
 
     async def _markfailed(self, appt, reason):
         now = self._getNowTick()

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/ast.py

@@ -3017,11 +3017,11 @@ class ArrayCond(Cond):
 
     async def getCondEval(self, runt):
 
-        name = await self.kids[0].compute(runt, None)
         cmpr = await self.kids[1].compute(runt, None)
 
         async def cond(node, path):
 
+            name = await self.kids[0].compute(runt, None)
             prop = node.form.props.get(name)
             if prop is None:
                 raise self.kids[0].addExcInfo(s_exc.NoSuchProp.init(name))

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/hiveauth.py

@@ -1,7 +1,7 @@
 import logging
 import dataclasses
 
-from typing import Union
+from typing import Optional, Union
 
 import synapse.exc as s_exc
 import synapse.common as s_common
@@ -941,16 +941,38 @@ class HiveUser(HiveRuler):
         if not self.allowed(perm):
             await self.addRule((True, perm), indx=0)
 
-    def allowed(self, perm, default=None, gateiden=None):
+    def allowed(self,
+                perm: tuple[str, ...],
+                default: Optional[str] = None,
+                gateiden: Optional[str] = None,
+                deepdeny: bool = False) -> Union[bool, None]:
+        '''
+        Check if a user is allowed a given permission.
+
+        Args:
+            perm: The permission tuple to check.
+            default: The default rule value if there is no match.
+            gateiden: The gate iden to check against.
+            deepdeny: If True, give precedence for checking deny rules which are more specific than the requested
+                      permission.
+
+        Notes:
+            The use of the deepdeny argument is intended for checking a less-specific part of a permissions tree, in
+            order to know about possible short circuit options. Using it to check a more specific part may have
+            unintended results.
+
+        Returns:
+            The allowed value of the permission.
+        '''
         perm = tuple(perm)
-        return self.permcache.get((perm, default, gateiden))
+        return self.permcache.get((perm, default, gateiden, deepdeny))
 
     def _allowed(self, pkey):
         '''
         NOTE: This must remain in sync with any changes to _getAllowedReason()!
         '''
 
-        perm, default, gateiden = pkey
+        perm, default, gateiden, deepdeny = pkey
 
         if self.info.get('locked'):
             return False
@@ -958,6 +980,9 @@ class HiveUser(HiveRuler):
         if self.info.get('admin'):
             return True
 
+        if deepdeny and self._hasDeepDeny(perm, gateiden):
+            return False
+
         # 1. check authgate user rules
         if gateiden is not None:
 
@@ -1055,6 +1080,58 @@ class HiveUser(HiveRuler):
 
         return _allowedReason(default, default=True)
 
+    def _hasDeepDeny(self, perm, gateiden):
+
+        permlen = len(perm)
+
+        # 1. check authgate user rules
+        if gateiden is not None:
+
+            info = self.authgates.get(gateiden)
+            if info is not None:
+
+                if info.get('admin'):
+                    return False
+
+                for allow, path in info.get('rules', ()):
+                    if allow:
+                        continue
+                    if path[:permlen] == perm and len(path) > permlen:
+                        return True
+
+        # 2. check user rules
+        for allow, path in self.info.get('rules', ()):
+            if allow:
+                continue
+
+            if path[:permlen] == perm and len(path) > permlen:
+                return True
+
+        # 3. check authgate role rules
+        if gateiden is not None:
+
+            for role in self.getRoles():
+
+                info = role.authgates.get(gateiden)
+                if info is None:
+                    continue
+
+                for allow, path in info.get('rules', ()):
+                    if allow:
+                        continue
+                    if path[:permlen] == perm and len(path) > permlen:
+                        return True
+
+        # 4. check role rules
+        for role in self.getRoles():
+            for allow, path in role.info.get('rules', ()):
+                if allow:
+                    continue
+                if path[:permlen] == perm and len(path) > permlen:
+                    return True
+
+        return False
+
     def clearAuthCache(self):
         self.permcache.clear()
         self.allowedcache.clear()

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/layer.py

@@ -4095,9 +4095,6 @@ class Layer(s_nexus.Pusher):
             yield prop[0]
 
     async def confirmLayerEditPerms(self, user, gateiden, delete=False):
-        if user.allowed(('node',), gateiden=gateiden):
-            return
-
         if delete:
             perm_forms = ('node', 'del')
             perm_props = ('node', 'prop', 'del')
@@ -4111,11 +4108,14 @@ class Layer(s_nexus.Pusher):
             perm_ndata = ('node', 'data', 'set')
             perm_edges = ('node', 'edge', 'add')
 
-        allow_forms = user.allowed(perm_forms, gateiden=gateiden)
-        allow_props = user.allowed(perm_props, gateiden=gateiden)
-        allow_tags = user.allowed(perm_tags, gateiden=gateiden)
-        allow_ndata = user.allowed(perm_ndata, gateiden=gateiden)
-        allow_edges = user.allowed(perm_edges, gateiden=gateiden)
+        if user.allowed(('node',), gateiden=gateiden, deepdeny=True):
+            return
+
+        allow_forms = user.allowed(perm_forms, gateiden=gateiden, deepdeny=True)
+        allow_props = user.allowed(perm_props, gateiden=gateiden, deepdeny=True)
+        allow_tags = user.allowed(perm_tags, gateiden=gateiden, deepdeny=True)
+        allow_ndata = user.allowed(perm_ndata, gateiden=gateiden, deepdeny=True)
+        allow_edges = user.allowed(perm_edges, gateiden=gateiden, deepdeny=True)
 
         if all((allow_forms, allow_props, allow_tags, allow_ndata, allow_edges)):
             return

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/lmdbslab.py

@@ -1623,7 +1623,17 @@ class Slab(s_base.Base):
         self.commitstats.append((starttime, xactopslen, delta))
 
         if self.WARN_COMMIT_TIME_MS and delta > self.WARN_COMMIT_TIME_MS:
-            logger.warning(f'Commit with {xactopslen} items in {self!r} took {delta} ms.')
+
+            extra = {
+                'delta': delta,
+                'path': self.path,
+                'sysctls': s_thisplat.getSysctls(),
+                'xactopslen': xactopslen,
+            }
+
+            mesg = f'Commit with {xactopslen} items in {self!r} took {delta} ms - performance may be degraded.'
+            logger.warning(mesg, extra={'synapse': extra})
+
         self._initCoXact()
         return True
 

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/modelrev.py

@@ -8,7 +8,7 @@ import synapse.lib.layer as s_layer
 
 logger = logging.getLogger(__name__)
 
-maxvers = (0, 2, 24)
+maxvers = (0, 2, 25)
 
 class ModelRev:
 
@@ -38,6 +38,7 @@ class ModelRev:
             ((0, 2, 22), self.revModel_0_2_22),
             ((0, 2, 23), self.revModel_0_2_23),
             ((0, 2, 24), self.revModel_0_2_24),
+            ((0, 2, 25), self.revModel_0_2_25),
         )
 
     async def _uniqSortArray(self, todoprops, layers):
@@ -757,6 +758,17 @@ class ModelRev:
         for form, props in formprops.items():
             await self._normVelocityProps(layers, form, props)
 
+    async def revModel_0_2_25(self, layers):
+        await self._typeToForm(layers, 'econ:currency', 'econ:currency')
+        await self._normPropValu(layers, 'ou:position:title')
+        await self._propToForm(layers, 'ou:position:title', 'ou:jobtitle')
+
+        await self._normPropValu(layers, 'ou:conference:name')
+        await self._propToForm(layers, 'ou:conference:name', 'entity:name')
+
+        await self._normPropValu(layers, 'ou:conference:names')
+        await self._propArrayToForm(layers, 'ou:conference:names', 'entity:name')
+
     async def runStorm(self, text, opts=None):
         '''
         Run storm code in a schedcoro and log the output messages.
@@ -1116,6 +1128,10 @@ class ModelRev:
         '''
         await self.runStorm(storm, opts=opts)
 
+    async def _typeToForm(self, layers, typename, formname):
+        for prop in layers[0].core.model.getPropsByType(typename):
+            await self._propToForm(layers, prop.full, formname)
+
     async def _propArrayToForm(self, layers, propfull, formname):
 
         opts = {'vars': {

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/modules.py

@@ -17,6 +17,7 @@ coremods = (
     'synapse.models.telco.TelcoModule',
     'synapse.models.inet.InetModule',
     'synapse.models.material.MatModule',
+    'synapse.models.entity.EntityModule',
     'synapse.models.language.LangModule',
     'synapse.models.crypto.CryptoModule',
     'synapse.models.gov.cn.GovCnModule',

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/msgpack.py

@@ -7,8 +7,27 @@ import synapse.exc as s_exc
 
 logger = logging.getLogger(__name__)
 
+def _ext_un(code, byts):
+    if code == 0:
+        return int.from_bytes(byts, 'big')
+    elif code == 1:
+        return int.from_bytes(byts, 'big', signed=True)
+    else:  # pragma: no cover
+        mesg = f'Invalid msgpack ext code: {code} ({repr(byts)[:20]})'
+        raise s_exc.SynErr(mesg=mesg)
+
+def _ext_en(item):
+    if isinstance(item, int):
+        if item > 0xffffffffffffffff:
+            size = (item.bit_length() + 7) // 8
+            return msgpack.ExtType(0, item.to_bytes(size, 'big'))
+        if item < -0x8000000000000000:
+            size = (item.bit_length() // 8) + 1
+            return msgpack.ExtType(1, item.to_bytes(size, 'big', signed=True))
+    return item
+
 # Single Packer object which is reused for performance
-pakr = msgpack.Packer(use_bin_type=True, unicode_errors='surrogatepass')
+pakr = msgpack.Packer(use_bin_type=True, unicode_errors='surrogatepass', default=_ext_en)
 if isinstance(pakr, m_fallback.Packer):  # pragma: no cover
     logger.warning('******************************************************************************************************')
     logger.warning('* msgpack is using the pure python fallback implementation. This will impact performance negatively. *')
@@ -21,6 +40,7 @@ unpacker_kwargs = {
     'raw': False,
     'use_list': False,
     'strict_map_key': False,
+    'ext_hook': _ext_un,
     'max_buffer_size': 2**32 - 1,
     'unicode_errors': 'surrogatepass'
 }
@@ -67,7 +87,8 @@ def _fallback_en(item):
         bytes: The serialized bytes in msgpack format.
     '''
     try:
-        return msgpack.packb(item, use_bin_type=True, unicode_errors='surrogatepass')
+        return msgpack.packb(item, use_bin_type=True,
+                             unicode_errors='surrogatepass', default=_ext_en)
     except TypeError as e:
         mesg = f'{e.args[0]}: {repr(item)[:20]}'
         raise s_exc.NotMsgpackSafe(mesg=mesg) from e
@@ -95,7 +116,8 @@ def un(byts, use_list=False):
         obj: The de-serialized object
     '''
     # This uses a subset of unpacker_kwargs
-    return msgpack.loads(byts, use_list=use_list, raw=False, strict_map_key=False, unicode_errors='surrogatepass')
+    return msgpack.loads(byts, use_list=use_list, raw=False, strict_map_key=False,
+                         unicode_errors='surrogatepass', ext_hook=_ext_un)
 
 def isok(item):
     '''

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/nexus.py

@@ -12,7 +12,6 @@ import synapse.common as s_common
 import synapse.telepath as s_telepath
 
 import synapse.lib.base as s_base
-import synapse.lib.version as s_version
 
 logger = logging.getLogger(__name__)
 
@@ -267,7 +266,7 @@ class NexsRoot(s_base.Base):
             raise
 
         except Exception:
-            logger.exception('Exception while replaying log')
+            logger.exception(f'Exception while replaying log: {s_common.trimText(repr(indxitem))}')
 
     async def addWriteHold(self, reason):
 
@@ -497,27 +496,32 @@ class NexsRoot(s_base.Base):
 
     async def runMirrorLoop(self, proxy):
 
-        cellinfo = await proxy.getCellInfo()
-        features = cellinfo.get('features', {})
-        if features.get('dynmirror'):
-            await proxy.readyToMirror()
+        try:
+            cellinfo = await proxy.getCellInfo()
+            features = cellinfo.get('features', {})
+            if features.get('dynmirror'):
+                await proxy.readyToMirror()
+
+            cellvers = cellinfo['cell']['version']
+            if cellvers > self.cell.VERSION:
+                logger.error('Leader is a higher version than we are. Mirrors must be updated first. Entering read-only mode.')
+                await self.addWriteHold(leaderversion)
+                # this will fire again on reconnect...
+                return
 
-        cellvers = cellinfo['cell']['version']
-        if cellvers > self.cell.VERSION:
-            logger.error('Leader is a higher version than we are. Mirrors must be updated first. Entering read-only mode.')
-            await self.addWriteHold(leaderversion)
-            # this will fire again on reconnect...
-            return
+            # When we reconnect and the leader version has become ok...
+            await self.delWriteHold(leaderversion)
 
-        # When we reconnect and the leader version has become ok...
-        await self.delWriteHold(leaderversion)
+            if self.celliden is not None:
+                if self.celliden != await proxy.getCellIden():
+                    logger.error('remote cell has different iden!  Aborting mirror sync')
+                    await proxy.fini()
+                    await self.fini()
+                    return
 
-        if self.celliden is not None:
-            if self.celliden != await proxy.getCellIden():
-                logger.error('remote cell has different iden!  Aborting mirror sync')
-                await proxy.fini()
-                await self.fini()
-                return
+        except Exception as exc:
+            logger.exception(f'Unknown error during mirror loop startup: {exc}')
+            await proxy.fini()
 
         while not proxy.isfini:
 
@@ -573,8 +577,8 @@ class NexsRoot(s_base.Base):
                         if respfutu is not None:
                             respfutu.set_result(retn)
 
-            except Exception:  # pragma: no cover
-                logger.exception('error in mirror loop')
+            except Exception as exc:  # pragma: no cover
+                logger.exception(f'error in mirror loop: {exc}')
 
         # If we've left the mirror loop for some reason, we no longer know if we
         # will be in the realtime window or not. So we should try to set the ready

{synapse-2.172.0 → synapse-2.174.0}/synapse/lib/schemas.py

@@ -287,3 +287,34 @@ _stormPoolOptsSchema = {
     'additionalProperties': False,
 }
 reqValidStormPoolOpts = s_config.getJsValidator(_stormPoolOptsSchema)
+
+# These types are order sensitive
+_changelogTypes = {'migration': 'Automatic Migrations',
+                   'model': 'Model Changes',
+                   'feat': 'Features and Enhancements',
+                   'bug': 'Bugfixes',
+                   'doc': 'Improved documentation',
+                   'deprecation': 'Deprecations'}
+
+_changelogSchema = {
+    'type': 'object',
+    'properties': {
+        'type': {
+            'type': 'string',
+            'enum': list(_changelogTypes.keys()),
+        },
+        'desc': {
+            'type': 'string',
+            'minLength': 1,
+        },
+        'prs': {
+            'type': 'array',
+            'items': {
+                'type': 'integer',
+            }
+        }
+    },
+    'additionalProperties': False,
+    'required': ['type', 'desc']
+}
+_reqChanglogSchema = s_config.getJsValidator(_changelogSchema)