synapse 2.199.0__tar.gz → 2.201.0__tar.gz

{synapse-2.199.0/synapse.egg-info → synapse-2.201.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse
-Version: 2.199.0
+Version: 2.201.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -23,7 +23,7 @@ Requires-Dist: pyOpenSSL<24.3.0,>=24.0.0
 Requires-Dist: cryptography<44.0.0,>=43.0.1
 Requires-Dist: msgpack<1.2.0,>=1.0.5
 Requires-Dist: xxhash<3.6.0,>=1.4.4
-Requires-Dist: lmdb<1.5.0,>=1.2.1
+Requires-Dist: lmdb<1.7.0,>=1.2.1
 Requires-Dist: tornado<7.0.0,>=6.2.0
 Requires-Dist: regex>=2022.9.11
 Requires-Dist: PyYAML<6.1.0,>=5.4

{synapse-2.199.0 → synapse-2.201.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.199.0'
+version = '2.201.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -27,7 +27,7 @@ dependencies = [
     'cryptography>=43.0.1,<44.0.0',
     'msgpack>=1.0.5,<1.2.0',
     'xxhash>=1.4.4,<3.6.0',
-    'lmdb>=1.2.1,<1.5.0',
+    'lmdb>=1.2.1,<1.7.0',
     'tornado>=6.2.0,<7.0.0',
     'regex>=2022.9.11',
     'PyYAML>=5.4,<6.1.0',

{synapse-2.199.0 → synapse-2.201.0}/synapse/cortex.py

@@ -20,7 +20,6 @@ import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
 import synapse.lib.chop as s_chop
 import synapse.lib.coro as s_coro
-import synapse.lib.hive as s_hive
 import synapse.lib.view as s_view
 import synapse.lib.cache as s_cache
 import synapse.lib.const as s_const
@@ -855,7 +854,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
     cellapi = CoreApi
     viewapi = s_view.ViewApi
     layerapi = s_layer.LayerApi
-    hiveapi = s_hive.HiveApi
 
     viewctor = s_view.View.anit
     layrctor = s_layer.Layer.anit
@@ -4656,10 +4654,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         if not path:
             return await self.cellapi.anit(self, link, user)
 
-        if path[0] == 'hive' and user.isAdmin():
-            s_common.deprecated('Cortex /hive telepath path', curv='2.198.0', eolv='2.199.0')
-            return await self.hiveapi.anit(self.hive, user)
-
         if path[0] == 'layer':
 
             if len(path) == 1:

{synapse-2.199.0 → synapse-2.201.0}/synapse/datamodel.py

@@ -1,6 +1,7 @@
 '''
 An API to assist with the creation and enforcement of cortex data models.
 '''
+import sys
 import asyncio
 import logging
 import collections
@@ -15,6 +16,7 @@ import synapse.lib.cache as s_cache
 import synapse.lib.types as s_types
 import synapse.lib.dyndeps as s_dyndeps
 import synapse.lib.grammar as s_grammar
+import synapse.lib.msgpack as s_msgpack
 
 logger = logging.getLogger(__name__)
 
@@ -142,6 +144,9 @@ class Prop:
             async def depfunc(node, oldv):
                 mesg = f'The property {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
                 await node.snap.warnonce(mesg)
+                if __debug__:
+                    sys.audit('synapse.datamodel.Prop.deprecated', mesg, self.full)
+
             self.onSet(depfunc)
 
     def __repr__(self):
@@ -306,6 +311,8 @@ class Form:
             async def depfunc(node):
                 mesg = f'The form {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
                 await node.snap.warnonce(mesg)
+                if __debug__:
+                    sys.audit('synapse.datamodel.Form.deprecated', mesg, self.full)
             self.onAdd(depfunc)
 
     def getStorNode(self, form):
@@ -1107,7 +1114,7 @@ class Model:
 
     def _prepFormIface(self, form, iface):
 
-        template = iface.get('template', {})
+        template = s_msgpack.deepcopy(iface.get('template', {}))
         template.update(form.type.info.get('template', {}))
 
         def convert(item):
@@ -1153,10 +1160,9 @@ class Model:
         for propname, typedef, propinfo in iface.get('props', ()):
 
             # allow form props to take precedence
-            if form.prop(propname) is not None:
-                continue
+            if (prop := form.prop(propname)) is None:
+                prop = self._addFormProp(form, propname, typedef, propinfo)
 
-            prop = self._addFormProp(form, propname, typedef, propinfo)
             self.ifaceprops[f'{name}:{propname}'].append(prop.full)
 
             if subifaces is not None:

{synapse-2.199.0 → synapse-2.201.0}/synapse/lib/agenda.py

@@ -260,6 +260,7 @@ class _Appt:
         'created',
         'enabled',
         'errcount',
+        'loglevel',
         'nexttime',
         'lasterrs',
         'isrunning',
@@ -269,7 +270,7 @@ class _Appt:
         'lastfinishtime',
     }
 
-    def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False):
+    def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False, loglevel=None):
         self.doc = ''
         self.name = ''
         self.task = None
@@ -284,6 +285,7 @@ class _Appt:
         self._recidxnexttime = None  # index of rec who is up next
         self.view = view
         self.created = created
+        self.loglevel = loglevel
 
         if self.recur and not self.recs:
             raise s_exc.BadTime(mesg='A recurrent appointment with no records')
@@ -364,7 +366,10 @@ class _Appt:
         if val['ver'] != 1:
             raise s_exc.BadStorageVersion(mesg=f"Found version {val['ver']}")  # pragma: no cover
         recs = [ApptRec.unpack(tupl) for tupl in val['recs']]
-        appt = cls(stor, val['iden'], val['recur'], val['indx'], val['query'], val['creator'], recs, nexttime=val['nexttime'], view=val.get('view'))
+        # TODO: MOAR INSANITY
+        loglevel = val.get('loglevel', 'WARNING')
+        appt = cls(stor, val['iden'], val['recur'], val['indx'], val['query'], val['creator'], recs,
+                   nexttime=val['nexttime'], view=val.get('view'), loglevel=loglevel)
         appt.doc = val.get('doc', '')
         appt.name = val.get('name', '')
         appt.pool = val.get('pool', False)
@@ -373,6 +378,7 @@ class _Appt:
         appt.lastfinishtime = val['lastfinishtime']
         appt.lastresult = val['lastresult']
         appt.enabled = val['enabled']
+        appt.lasterrs = list(val.get('lasterrs', []))
 
         return appt
 
@@ -422,8 +428,10 @@ class _Appt:
                 logger.warning('_Appt.edits() Invalid attribute received: %s = %r', name, valu, extra=extra)
                 continue
 
-            else:
-                setattr(self, name, valu)
+            if name == 'lasterrs' and not isinstance(valu, list):
+                valu = list(valu)
+
+            setattr(self, name, valu)
 
         await self.save()
 
@@ -559,6 +567,7 @@ class Agenda(s_base.Base):
         creator = cdef.get('creator')
         view = cdef.get('view')
         created = cdef.get('created')
+        loglevel = cdef.get('loglevel', 'WARNING')
 
         pool = cdef.get('pool', False)
 
@@ -603,7 +612,9 @@ class Agenda(s_base.Base):
                 incvals = (incvals, )
             recs.extend(ApptRec(rd, incunit, v) for (rd, v) in itertools.product(reqdicts, incvals))
 
-        appt = _Appt(self, iden, recur, indx, query, creator, recs, nexttime=nexttime, view=view, created=created, pool=pool)
+        # TODO: this is insane. Make _Appt take the cdef directly...
+        appt = _Appt(self, iden, recur, indx, query, creator, recs, nexttime=nexttime, view=view,
+                           created=created, pool=pool, loglevel=loglevel)
         self._addappt(iden, appt)
 
         appt.doc = cdef.get('doc', '')
@@ -841,7 +852,10 @@ class Agenda(s_base.Base):
                     extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden, 'text': appt.query,
                                        'username': user.name, 'view': appt.view}})
         starttime = self._getNowTick()
+
         success = False
+        loglevel = s_common.normLogLevel(appt.loglevel)
+
         try:
             opts = {
                 'user': user.iden,
@@ -861,6 +875,11 @@ class Agenda(s_base.Base):
                 if mesg[0] == 'node':
                     count += 1
 
+                elif mesg[0] == 'warn' and loglevel <= logging.WARNING:
+                    text = mesg[1].get('mesg', '<missing message>')
+                    extra = await self.core.getLogExtra(cron=appt.iden, **mesg[1])
+                    logger.warning(f'Cron job {appt.iden} issued warning: {text}', extra=extra)
+
                 elif mesg[0] == 'err':
                     excname, errinfo = mesg[1]
                     errinfo.pop('eline', None)

{synapse-2.199.0 → synapse-2.201.0}/synapse/lib/ast.py

@@ -217,13 +217,10 @@ class Query(AstNode):
                 genr = await stack.enter_async_context(contextlib.aclosing(oper.run(runt, genr)))
 
             async for node, path in genr:
-                runt.tick()
                 yield node, path
 
     async def iterNodePaths(self, runt, genr=None):
 
-        count = 0
-
         self.optimize()
         self.validate(runt)
 
@@ -231,18 +228,18 @@ class Query(AstNode):
         if genr is None:
             genr = runt.getInput()
 
+        count = 0
+        limit = runt.getOpt('limit')
+
         async with contextlib.aclosing(self.run(runt, genr)) as agen:
             async for node, path in agen:
 
-                runt.tick()
-
                 yield node, path
 
-                count += 1
-
-                limit = runt.getOpt('limit')
-                if limit is not None and count >= limit:
-                    break
+                if limit is not None:
+                    count += 1
+                    if count >= limit:
+                        break
 
 class Lookup(Query):
     '''

{synapse-2.199.0 → synapse-2.201.0}/synapse/lib/hive.py

@@ -86,7 +86,7 @@ class Node(s_base.Base):
         for name, node in self.kids.items():
             yield name, node
 
-class Hive(s_nexus.Pusher, s_telepath.Aware):
+class Hive(s_nexus.Pusher):
     '''
     An optionally persistent atomically accessed tree which implements
     primitives for use in making distributed/clustered services.
@@ -95,8 +95,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
         await s_nexus.Pusher.__anit__(self, 'hive', nexsroot=nexsroot)
 
-        s_telepath.Aware.__init__(self)
-
         if conf is None:
             conf = {}
 
@@ -161,24 +159,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
             for cullname in culls:
                 await node.pop((cullname,))
 
-    async def getHiveAuth(self):
-        '''
-        Retrieve a HiveAuth for hive standalone or non-cell uses.
-
-        Note:
-            This is for the hive's own auth, or for non-cell auth.  It isn't the same auth as for a cell
-        '''
-        import synapse.lib.hiveauth as s_hiveauth
-        if self.auth is None:
-
-            path = tuple(self.conf.get('auth:path').split('/'))
-
-            node = await self.open(path)
-            self.auth = await s_hiveauth.Auth.anit(node, nexsroot=self.nexsroot)
-            self.onfini(self.auth.fini)
-
-        return self.auth
-
     async def _onHiveFini(self):
         await self.root.fini()
 
@@ -423,27 +403,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
 
         return node.valu
 
-    async def getTeleApi(self, link, mesg, path):
-        s_common.deprecated('Hive.getTeleApi', curv='2.198.0', eolv='2.199.0')
-        auth = await self.getHiveAuth()
-
-        if not self.conf.get('auth:en'):
-            user = await auth.getUserByName('root')
-            return await HiveApi.anit(self, user)
-
-        name, info = mesg[1].get('auth')
-
-        user = await auth.getUserByName(name)
-        if user is None:
-            raise s_exc.NoSuchUser(name=name)
-
-        # passwd None always fails...
-        passwd = info.get('passwd')
-        if not await user.tryPasswd(passwd):
-            raise s_exc.AuthDeny(mesg='Invalid password', user=user.iden, username=user.name)
-
-        return await HiveApi.anit(self, user)
-
     async def _storLoadHive(self):
         pass
 
@@ -480,224 +439,6 @@ class SlabHive(Hive):
         lkey = '\x00'.join(full).encode('utf8')
         self.slab.pop(lkey, db=self.db)
 
-class HiveApi(s_base.Base):
-
-    async def __anit__(self, hive, user):
-
-        await s_base.Base.__anit__(self)
-
-        self.hive = hive
-        self.user = user
-
-        self.msgq = asyncio.Queue(maxsize=10000)
-
-        self.onfini(self._onHapiFini)
-
-    async def loadHiveTree(self, tree, path=(), trim=False):
-        s_common.deprecated('HiveApi.loadHiveTree', curv='2.167.0')
-        return await self.hive.loadHiveTree(tree, path=path, trim=trim)
-
-    async def saveHiveTree(self, path=()):
-        s_common.deprecated('HiveApi.saveHiveTree', curv='2.167.0')
-        return await self.hive.saveHiveTree(path=path)
-
-    async def treeAndSync(self, path, iden):
-        s_common.deprecated('HiveApi.treeAndSync', curv='2.167.0')
-
-        node = await self.hive.open(path)
-
-        # register handlers...
-        node.on('hive:add', self._onHiveEdit, base=self)
-        node.on('hive:set', self._onHiveEdit, base=self)
-        node.on('hive:pop', self._onHiveEdit, base=self)
-
-        # serialize the subtree into a message and return
-        # via the mesg queue so there is no get/update race
-        root = (node.valu, {})
-
-        todo = collections.deque([(node, root)])
-
-        # breadth first generator
-        while todo:
-
-            node, pode = todo.popleft()
-
-            for name, kidn in node.kids.items():
-
-                kidp = (kidn.valu, {})
-                pode[1][name] = kidp
-
-                todo.append((kidn, kidp))
-
-        await self.msgq.put(('hive:tree', {'path': path, 'tree': root}))
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return
-
-    async def setAndSync(self, path, valu, iden, nexs=False):
-        s_common.deprecated('HiveApi.setAndSync', curv='2.167.0')
-
-        valu = await self.hive.set(path, valu, nexs=nexs)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def addAndSync(self, path, valu, iden):
-        s_common.deprecated('HiveApi.addAndSync', curv='2.167.0')
-
-        valu = await self.hive.add(path, valu)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def popAndSync(self, path, iden, nexs=False):
-        s_common.deprecated('HiveApi.popAndSync', curv='2.167.0')
-
-        valu = await self.hive.pop(path, nexs=nexs)
-        await self.msgq.put(('hive:sync', {'iden': iden}))
-        return valu
-
-    async def _onHapiFini(self):
-        await self.msgq.put(None)
-
-    async def _onHiveEdit(self, mesg):
-        self.msgq.put_nowait(mesg)
-
-    async def get(self, full):
-        s_common.deprecated('HiveApi.get', curv='2.167.0')
-        return await self.hive.get(full)
-
-    async def edits(self):
-        s_common.deprecated('HiveApi.edits', curv='2.167.0')
-
-        while not self.isfini:
-
-            item = await self.msgq.get()
-            if item is None:
-                return
-
-            yield item
-
-class TeleHive(Hive):
-    '''
-    A Hive that acts as a consistent read cache for a telepath proxy Hive
-    '''
-
-    async def __anit__(self, proxy):
-
-        self.proxy = proxy
-
-        await Hive.__anit__(self)
-
-        self.lock = asyncio.Lock()
-
-        self.syncevents = {}  # iden: asyncio.Event()
-
-        # fire a task to sync the sections of the tree we open
-        self.schedCoro(self._runHiveLoop())
-
-        self.mesgbus = await s_base.Base.anit()
-        self.mesgbus.on('hive:set', self._onHiveSet)
-        self.mesgbus.on('hive:pop', self._onHivePop)
-        self.mesgbus.on('hive:tree', self._onHiveTree)
-        self.mesgbus.on('hive:sync', self._onHiveSync)
-
-        self.onfini(self.mesgbus.fini)
-
-        self.onfini(proxy.fini)
-
-    async def _onHiveSync(self, mesg):
-
-        iden = mesg[1].get('iden')
-        evnt = self.syncevents.pop(iden, None)
-        if evnt is None:
-            return
-
-        evnt.set()
-
-    def _getSyncIden(self):
-        iden = s_common.guid()
-        evnt = asyncio.Event()
-        self.syncevents[iden] = evnt
-        return iden, evnt
-
-    async def _runHiveLoop(self):
-        while not self.isfini:
-            async for mesg in self.proxy.edits():
-                await self.mesgbus.dist(mesg)
-
-    async def _onHiveSet(self, mesg):
-        path = mesg[1].get('path')
-        valu = mesg[1].get('valu')
-        await Hive.set(self, path, valu)
-
-    async def _onHivePop(self, mesg):
-        path = mesg[1].get('path')
-        await Hive.pop(self, path)
-
-    async def _onHiveTree(self, mesg):
-
-        # get an entire tree update at once
-        path = mesg[1].get('path')
-        tree = mesg[1].get('tree')
-
-        node = await Hive.open(self, path)
-
-        todo = collections.deque([(node, path, tree)])
-
-        while todo:
-
-            node, path, (valu, kids) = todo.popleft()
-
-            # do *not* go through the set() API
-            node.valu = valu
-            for name, kidt in kids.items():
-
-                kidp = path + (name,)
-                kidn = await Hive.open(self, kidp)
-
-                todo.append((kidn, kidp, kidt))
-
-    async def set(self, path, valu, nexs=False):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.setAndSync(path, valu, iden, nexs=nexs)
-        await evnt.wait()
-        return valu
-
-    async def add(self, path, valu):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.addAndSync(path, valu, iden)
-        await evnt.wait()
-        return valu
-
-    async def pop(self, path, nexs=False):
-        iden, evnt = self._getSyncIden()
-        valu = await self.proxy.popAndSync(path, iden, nexs=nexs)
-        await evnt.wait()
-        return valu
-
-    async def get(self, path):
-        return await self.proxy.get(path)
-
-    async def open(self, path):
-
-        # try once pre-lock for speed
-        node = self.nodes.get(path)
-        if node is not None:
-            return node
-
-        async with self.lock:
-
-            # try again with lock to avoid race
-            node = self.nodes.get(path)
-            if node is not None:
-                return node
-
-            iden, evnt = self._getSyncIden()
-
-            await self.proxy.treeAndSync(path, iden)
-
-            await evnt.wait()
-
-            return self.nodes.get(path)
-
 class HiveDict(s_base.Base):
     '''
     '''
@@ -758,11 +499,6 @@ def iterpath(path):
     for i in range(len(path)):
         yield path[:i + 1]
 
-async def openurl(url, **opts):
-    s_common.deprecated('synapse.lib.hive.openurl()', curv='2.198.0', eolv='2.199.0')
-    prox = await s_telepath.openurl(url, **opts)
-    return await TeleHive.anit(prox)
-
 async def opendir(dirn, conf=None):
     slab = await s_slab.Slab.anit(dirn, map_size=s_const.gibibyte)
     db = slab.initdb('hive')

{synapse-2.199.0 → synapse-2.201.0}/synapse/lib/schemas.py

@@ -97,6 +97,7 @@ _CronJobSchema = {
         'name': {'type': 'string'},
         'pool': {'type': 'boolean'},
         'doc': {'type': 'string'},
+        'loglevel': {'type': 'string', 'enum': list(s_const.LOG_LEVEL_CHOICES.keys())},
         'incunit': {
             'oneOf': [
                 {'type': 'null'},

{synapse-2.199.0 → synapse-2.201.0}/synapse/lib/snap.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import sys
 import types
 import asyncio
 import logging
@@ -12,14 +11,12 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.base as s_base
-import synapse.lib.coro as s_coro
 import synapse.lib.node as s_node
 import synapse.lib.time as s_time
 import synapse.lib.cache as s_cache
 import synapse.lib.layer as s_layer
 import synapse.lib.storm as s_storm
 import synapse.lib.types as s_types
-import synapse.lib.spooled as s_spooled
 
 logger = logging.getLogger(__name__)
 
@@ -1606,7 +1603,8 @@ class Snap(s_base.Base):
         proptype = prop.type
         for prop in prop.getAlts():
             if prop.type.isarray and prop.type.arraytype == proptype:
-                if valu in node.get(prop.name):
+                arryvalu = node.get(prop.name)
+                if arryvalu is not None and valu in arryvalu:
                     return True
             else:
                 if node.get(prop.name) == valu:
@@ -1694,10 +1692,6 @@ class Snap(s_base.Base):
         return tagnode
 
     async def _raiseOnStrict(self, ctor, mesg, **info):
-        if __debug__:
-            if issubclass(ctor, s_exc.IsDeprLocked):
-                sys.audit('synapse.exc.IsDeprLocked', (mesg, info))
-
         if self.strict:
             raise ctor(mesg=mesg, **info)
         await self.warn(mesg)