synapse 2.195.1__py311-none-any.whl → 2.196.0__py311-none-any.whl

synapse/axon.py

@@ -1,5 +1,6 @@
 import csv
 import json
+import struct
 import asyncio
 import hashlib
 import logging
@@ -759,6 +760,26 @@ class AxonApi(s_cell.CellApi, s_share.Share):  # type: ignore
         async for item in self.cell.jsonlines(sha256, errors=errors):
             yield item
 
+    async def unpack(self, sha256, fmt, offs=0):
+        '''
+        Unpack bytes from a file in the Axon using struct.
+
+        Args:
+            sha256 (bytes): The sha256 hash of the file in bytes.
+            fmt (str): The struct format string.
+            offs (int): The offset to start reading from.
+
+        Returns:
+            tuple: The unpacked values.
+
+        Raises:
+            synapse.exc.NoSuchFile: If the file does not exist.
+            synapse.exc.BadArg: If the struct format is invalid or reads too much data.
+            synapse.exc.BadDataValu: If the file does not contain the expected number of bytes.
+            synapse.exc.FeatureNotSupported: Feature is not supported.
+        '''
+        await self._reqUserAllowed(('axon', 'get'))
+        return await self.cell.unpack(sha256, fmt, offs=offs)
 
 class Axon(s_cell.Cell):
 
@@ -816,6 +837,15 @@ class Axon(s_cell.Cell):
         # modularize blob storage
         await self._initBlobStor()
 
+        # Set the byterange flag as an integer AFTER we've called
+        # _initBlobStor which may set it to true. That will allow
+        # downstream implementations to continue working as expected
+        # out of the gate.
+        self.features.update({
+            'byterange': int(self.byterange),
+            'unpack': 1,
+        })
+
     async def initServiceRuntime(self):
 
         # share ourself via the cell dmon as "axon"
@@ -825,11 +855,6 @@ class Axon(s_cell.Cell):
         self._initAxonHttpApi()
         self.addHealthFunc(self._axonHealth)
 
-    async def getCellInfo(self):
-        info = await s_cell.Cell.getCellInfo(self)
-        info['features']['byterange'] = self.byterange
-        return info
-
     @contextlib.asynccontextmanager
     async def holdHashLock(self, hashbyts):
         '''
@@ -1487,6 +1512,48 @@ class Axon(s_cell.Cell):
                 raise s_exc.BadJsonText(mesg=f'Bad json line encountered while processing {sha256}, ({e})',
                                         sha256=sha256) from None
 
+    async def unpack(self, sha256, fmt, offs=0):
+        '''
+        Unpack bytes from a file in the Axon using struct.
+
+        Args:
+            sha256 (bytes): The sha256 hash of the file in bytes.
+            fmt (str): The struct format string.
+            offs (int): The offset to start reading from.
+
+        Returns:
+            tuple: The unpacked values.
+
+        Raises:
+            synapse.exc.NoSuchFile: If the file does not exist.
+            synapse.exc.BadArg: If the struct format is invalid.
+            synapse.exc.BadDataValu: If the expected number of bytes is not received.
+            synapse.exc.FeatureNotSupported: Feature is not supported.
+        '''
+
+        if not isinstance(fmt, str):
+            raise s_exc.BadArg(mesg='Format string must be a string', fmt=fmt)
+
+        try:
+            size = struct.calcsize(fmt)
+
+            if size > s_const.mebibyte:
+                raise s_exc.BadArg(mesg=f'Struct format would read too much data: {size} bytes', size=size)
+
+            byts = b''
+            async for chunk in self.get(sha256, offs=offs, size=size):
+                byts += chunk
+
+            if len(byts) != size:
+                mesg = f'Expected {size} bytes but got {len(byts)} bytes'
+                raise s_exc.BadDataValu(mesg=mesg, expected=size, received=len(byts))
+
+            return struct.unpack(fmt, byts)
+
+        except struct.error as e:
+            mesg = f'Failed to unpack bytes with format {fmt}: {str(e)}'
+            raise s_exc.BadArg(mesg=mesg) from None
+
     async def postfiles(self, fields, url, params=None, headers=None, method='POST',
                         ssl=True, timeout=None, proxy=True, ssl_opts=None):
         '''

synapse/common.py

@@ -1383,6 +1383,29 @@ def _timeout(delay):
     """
     loop = asyncio.get_running_loop()
     return _Timeout(loop.time() + delay if delay is not None else None)
+# End - Vendored Code from Python 3.12+
+
+async def waitretn(futu, timeout):
+    try:
+        valu = await wait_for(futu, timeout)
+        return (True, valu)
+    except Exception as e:
+        return (False, excinfo(e))
+
+async def waitgenr(genr, timeout):
+
+    async with contextlib.aclosing(genr.__aiter__()) as genr:
+
+        while True:
+            retn = await waitretn(genr.__anext__(), timeout)
+
+            if not retn[0] and retn[1]['err'] == 'StopAsyncIteration':
+                return
+
+            yield retn
+
+            if not retn[0]:
+                return
 
 def format(text, **kwargs):
     '''

synapse/cortex.py

@@ -83,6 +83,7 @@ import synapse.lib.stormlib.model as s_stormlib_model
 import synapse.lib.stormlib.oauth as s_stormlib_oauth  # NOQA
 import synapse.lib.stormlib.stats as s_stormlib_stats  # NOQA
 import synapse.lib.stormlib.storm as s_stormlib_storm  # NOQA
+import synapse.lib.stormlib.utils as s_stormlib_utils  # NOQA
 import synapse.lib.stormlib.vault as s_stormlib_vault  # NOQA
 import synapse.lib.stormlib.backup as s_stormlib_backup  # NOQA
 import synapse.lib.stormlib.cortex as s_stormlib_cortex  # NOQA

synapse/daemon.py

@@ -462,6 +462,7 @@ class Daemon(s_base.Base):
             link.set('sess', sess)
 
             if isinstance(item, s_telepath.Aware):
+                reply[1]['features'] = await item.getTeleFeats()
                 item = await s_coro.ornot(item.getTeleApi, link, mesg, path)
                 if isinstance(item, s_base.Base):
                     link.onfini(item)

synapse/lib/aha.py

@@ -132,10 +132,18 @@ class AhaApi(s_cell.CellApi):
 
     async def getAhaUrls(self, user='root'):
         ahaurls = await self.cell.getAhaUrls(user=user)
-        if ahaurls is None:
-            return ()
         return ahaurls
 
+    @s_cell.adminapi()
+    async def callAhaPeerApi(self, iden, todo, timeout=None, skiprun=None):
+        async for item in self.cell.callAhaPeerApi(iden, todo, timeout=timeout, skiprun=skiprun):
+            yield item
+
+    @s_cell.adminapi()
+    async def callAhaPeerGenr(self, iden, todo, timeout=None, skiprun=None):
+        async for item in self.cell.callAhaPeerGenr(iden, todo, timeout=timeout, skiprun=skiprun):
+            yield item
+
     async def getAhaSvc(self, name, filters=None):
         '''
         Return an AHA service description dictionary for a service name.
@@ -366,7 +374,6 @@ class AhaApi(s_cell.CellApi):
         '''
         return await self.cell.clearAhaClones()
 
-
 class ProvDmon(s_daemon.Daemon):
 
     async def __anit__(self, aha):
@@ -571,7 +578,8 @@ class AhaCell(s_cell.Cell):
 
     async def initServiceStorage(self):
 
-        # TODO plumb using a remote jsonstor?
+        self.features['callpeers'] = 1
+
         dirn = s_common.gendir(self.dirn, 'slabs', 'jsonstor')
 
         slab = await s_lmdbslab.Slab.anit(dirn)
@@ -682,8 +690,126 @@ class AhaCell(s_cell.Cell):
         self.addHttpApi('/api/v1/aha/services', AhaServicesV1, {'cell': self})
         self.addHttpApi('/api/v1/aha/provision/service', AhaProvisionServiceV1, {'cell': self})
 
+    async def callAhaSvcApi(self, name, todo, timeout=None):
+        name = self._getAhaName(name)
+        svcdef = await self._getAhaSvc(name)
+        return self._callAhaSvcApi(svcdef, todo, timeout=timeout)
+
+    async def _callAhaSvcApi(self, svcdef, todo, timeout=None):
+        try:
+            proxy = await self.getAhaSvcProxy(svcdef, timeout=timeout)
+            meth = getattr(proxy, todo[0])
+            return await s_common.waitretn(meth(*todo[1], **todo[2]), timeout=timeout)
+        except Exception as e:
+            # in case proxy construction fails
+            return (False, s_common.excinfo(e))
+
+    async def _callAhaSvcGenr(self, svcdef, todo, timeout=None):
+        try:
+            proxy = await self.getAhaSvcProxy(svcdef, timeout=timeout)
+            meth = getattr(proxy, todo[0])
+            async for item in s_common.waitgenr(meth(*todo[1], **todo[2]), timeout=timeout):
+                yield item
+        except Exception as e:
+            # in case proxy construction fails
+            yield (False, s_common.excinfo(e))
+
+    async def getAhaSvcsByIden(self, iden, online=True, skiprun=None):
+
+        runs = set()
+        async for svcdef in self.getAhaSvcs():
+            await asyncio.sleep(0)
+
+            # TODO services by iden indexes (SYN-8467)
+            if svcdef['svcinfo'].get('iden') != iden:
+                continue
+
+            if online and svcdef['svcinfo'].get('online') is None:
+                continue
+
+            svcrun = svcdef['svcinfo'].get('run')
+            if svcrun in runs:
+                continue
+
+            if skiprun == svcrun:
+                continue
+
+            runs.add(svcrun)
+            yield svcdef
+
+    def getAhaSvcUrl(self, svcdef, user='root'):
+        svcfull = svcdef.get('name')
+        svcnetw = svcdef.get('svcnetw')
+        host = svcdef['svcinfo']['urlinfo']['host']
+        port = svcdef['svcinfo']['urlinfo']['port']
+        return f'ssl://{host}:{port}?hostname={svcfull}&certname={user}@{svcnetw}'
+
+    async def callAhaPeerApi(self, iden, todo, timeout=None, skiprun=None):
+
+        if not self.isactive:
+            proxy = await self.nexsroot.client.proxy(timeout=timeout)
+            async for item in proxy.callAhaPeerApi(iden, todo, timeout=timeout, skiprun=skiprun):
+                yield item
+
+        queue = asyncio.Queue()
+        async with await s_base.Base.anit() as base:
+
+            async def call(svcdef):
+                svcfull = svcdef.get('name')
+                await queue.put((svcfull, await self._callAhaSvcApi(svcdef, todo, timeout=timeout)))
+
+            count = 0
+            async for svcdef in self.getAhaSvcsByIden(iden, skiprun=skiprun):
+                count += 1
+                base.schedCoro(call(svcdef))
+
+            for i in range(count):
+                yield await queue.get()
+
+    async def callAhaPeerGenr(self, iden, todo, timeout=None, skiprun=None):
+
+        if not self.isactive:
+            proxy = await self.nexsroot.client.proxy(timeout=timeout)
+            async for item in proxy.callAhaPeerGenr(iden, todo, timeout=timeout, skiprun=skiprun):
+                yield item
+
+        queue = asyncio.Queue()
+        async with await s_base.Base.anit() as base:
+
+            async def call(svcdef):
+                svcfull = svcdef.get('name')
+                try:
+                    async for item in self._callAhaSvcGenr(svcdef, todo, timeout=timeout):
+                        await queue.put((svcfull, item))
+                finally:
+                    await queue.put(None)
+
+            count = 0
+            async for svcdef in self.getAhaSvcsByIden(iden, skiprun=skiprun):
+                count += 1
+                base.schedCoro(call(svcdef))
+
+            while count > 0:
+
+                item = await queue.get()
+                if item is None:
+                    count -= 1
+                    continue
+
+                yield item
+
+    async def _finiSvcClients(self):
+        for client in list(self.clients.values()):
+            await client.fini()
+
+    async def initServicePassive(self):
+        await self._finiSvcClients()
+
     async def initServiceRuntime(self):
 
+        self.clients = {}
+        self.onfini(self._finiSvcClients)
+
         self.addActiveCoro(self._clearInactiveSessions)
 
         if self.isactive:
@@ -897,6 +1023,31 @@ class AhaCell(s_cell.Cell):
         await self.fire('aha:svcadd', svcinfo=svcinfo)
         await self.fire(f'aha:svcadd:{svcfull}', svcinfo=svcinfo)
 
+    async def getAhaSvcProxy(self, svcdef, timeout=None):
+
+        client = await self.getAhaSvcClient(svcdef)
+        if client is None:
+            return None
+
+        return await client.proxy(timeout=timeout)
+
+    async def getAhaSvcClient(self, svcdef):
+
+        svcfull = svcdef.get('name')
+
+        client = self.clients.get(svcfull)
+        if client is not None:
+            return client
+
+        svcurl = self.getAhaSvcUrl(svcdef)
+
+        client = self.clients[svcfull] = await s_telepath.ClientV2.anit(svcurl)
+        async def fini():
+            self.clients.pop(svcfull, None)
+
+        client.onfini(fini)
+        return client
+
     def _getAhaName(self, name):
         # the modern version of names is absolute or ...
         if name.endswith('...'):
@@ -1066,6 +1217,10 @@ class AhaCell(s_cell.Cell):
         logger.info(f'Set [{svcfull}] offline.',
                         extra=await self.getLogExtra(name=svcname, netw=svcnetw))
 
+        client = self.clients.pop(svcfull, None)
+        if client is not None:
+            await client.fini()
+
     async def getAhaSvc(self, name, filters=None):
 
         name = self._getAhaName(name)

synapse/lib/cell.py

@@ -83,6 +83,8 @@ permnames = {
     PERM_ADMIN: 'admin',
 }
 
+feat_aha_callpeers_v1 = ('callpeers', 1)
+
 diskspace = "Insufficient free space on disk."
 
 def adminapi(log=False):
@@ -435,6 +437,19 @@ class CellApi(s_base.Base):
     async def kill(self, iden):
         return await self.cell.kill(self.user, iden)
 
+    @adminapi()
+    async def getTasks(self, peers=True, timeout=None):
+        async for task in self.cell.getTasks(peers=peers, timeout=timeout):
+            yield task
+
+    @adminapi()
+    async def getTask(self, iden, peers=True, timeout=None):
+        return await self.cell.getTask(iden, peers=peers, timeout=timeout)
+
+    @adminapi()
+    async def killTask(self, iden, peers=True, timeout=None):
+        return await self.cell.killTask(iden, peers=peers, timeout=timeout)
+
     @adminapi(log=True)
     async def behold(self):
         '''
@@ -765,8 +780,8 @@ class CellApi(s_base.Base):
         return await self.cell.saveHiveTree(path=path)
 
     @adminapi()
-    async def getNexusChanges(self, offs, tellready=False):
-        async for item in self.cell.getNexusChanges(offs, tellready=tellready):
+    async def getNexusChanges(self, offs, tellready=False, wait=True):
+        async for item in self.cell.getNexusChanges(offs, tellready=tellready, wait=wait):
             yield item
 
     @adminapi()
@@ -1165,6 +1180,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         self.netready = asyncio.Event()
 
         self.conf = self._initCellConf(conf)
+        self.features = {
+            'tellready': 1,
+            'dynmirror': 1,
+            'tasks': 1,
+        }
 
         self.minfree = self.conf.get('limit:disk:free')
         if self.minfree is not None:
@@ -2382,8 +2402,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
     async def initServicePassive(self):  # pragma: no cover
         pass
 
-    async def getNexusChanges(self, offs, tellready=False):
-        async for item in self.nexsroot.iter(offs, tellready=tellready):
+    async def getNexusChanges(self, offs, tellready=False, wait=True):
+        async for item in self.nexsroot.iter(offs, tellready=tellready, wait=wait):
             yield item
 
     def _reqBackDirn(self, name):
@@ -4414,6 +4434,111 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
 
         return retn
 
+    async def getAhaProxy(self, timeout=None, feats=None):
+
+        if self.ahaclient is None:
+            return
+
+        proxy = await self.ahaclient.proxy(timeout=timeout)
+        if proxy is None:
+            logger.warning('AHA client connection failed.')
+            return
+
+        if feats is not None:
+            for name, vers in feats:
+                if not proxy._hasTeleFeat(name, vers):
+                    logger.warning(f'AHA server does not support feature: {name} >= {vers}')
+                    return None
+
+        return proxy
+
+    async def callPeerApi(self, todo, timeout=None):
+        '''
+        Yield responses from our peers via the AHA gather call API.
+        '''
+        proxy = await self.getAhaProxy(timeout=timeout, feats=(feat_aha_callpeers_v1,))
+        if proxy is None:
+            return
+
+        async for item in proxy.callAhaPeerApi(self.iden, todo, timeout=timeout, skiprun=self.runid):
+            yield item
+
+    async def callPeerGenr(self, todo, timeout=None):
+        '''
+        Yield responses from invoking a generator via the AHA gather API.
+        '''
+        proxy = await self.getAhaProxy(timeout=timeout, feats=(feat_aha_callpeers_v1,))
+        if proxy is None:
+            return
+
+        async for item in proxy.callAhaPeerGenr(self.iden, todo, timeout=timeout, skiprun=self.runid):
+            yield item
+
+    async def getTasks(self, peers=True, timeout=None):
+
+        for task in self.boss.ps():
+
+            item = task.pack()
+            item['service'] = self.ahasvcname
+
+            yield item
+
+        if not peers:
+            return
+
+        todo = s_common.todo('getTasks', peers=False)
+        # we can ignore the yielded aha names because we embed it in the task
+        async for (ahasvc, (ok, retn)) in self.callPeerGenr(todo, timeout=timeout):
+
+            if not ok: # pragma: no cover
+                logger.warning(f'getTasks() on {ahasvc} failed: {retn}')
+                continue
+
+            yield retn
+
+    async def getTask(self, iden, peers=True, timeout=None):
+
+        task = self.boss.get(iden)
+        if task is not None:
+            item = task.pack()
+            item['service'] = self.ahasvcname
+            return item
+
+        if not peers:
+            return
+
+        todo = s_common.todo('getTask', iden, peers=False, timeout=timeout)
+        async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout):
+
+            if not ok: # pragma: no cover
+                logger.warning(f'getTask() on {ahasvc} failed: {retn}')
+                continue
+
+            if retn is not None:
+                return retn
+
+    async def killTask(self, iden, peers=True, timeout=None):
+
+        task = self.boss.get(iden)
+        if task is not None:
+            await task.kill()
+            return True
+
+        if not peers:
+            return False
+
+        todo = s_common.todo('killTask', iden, peers=False, timeout=timeout)
+        async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout):
+
+            if not ok: # pragma: no cover
+                logger.warning(f'killTask() on {ahasvc} failed: {retn}')
+                continue
+
+            if retn:
+                return True
+
+        return False
+
     async def kill(self, user, iden):
         perm = ('task', 'del')
         isallowed = await self.isUserAllowed(user.iden, perm)
@@ -4488,13 +4613,13 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
                     'https': self.https_listeners,
                 }
             },
-            'features': {
-                'tellready': True,
-                'dynmirror': True,
-            },
+            'features': self.features,
         }
         return ret
 
+    async def getTeleFeats(self):
+        return dict(self.features)
+
     async def getSystemInfo(self):
         '''
         Get info about the system in which the cell is running

synapse/lib/jsonstor.py

@@ -234,7 +234,8 @@ class JsonStor(s_base.Base):
         step = item
         names = self._pathToTupl(prop)
         for name in names[:-1]:
-            step = step[name]
+            if (step := step.get(name)) is None:
+                return False
 
         step.pop(names[-1], None)
 

synapse/lib/modelrev.py

@@ -13,7 +13,7 @@ import synapse.models.infotech as s_infotech
 
 logger = logging.getLogger(__name__)
 
-maxvers = (0, 2, 32)
+maxvers = (0, 2, 33)
 
 class ModelRev:
 
@@ -51,6 +51,7 @@ class ModelRev:
             ((0, 2, 30), self.revModel_0_2_30),
             ((0, 2, 31), self.revModel_0_2_31),
             ((0, 2, 32), self.revModel_0_2_32),
+            ((0, 2, 33), self.revModel_0_2_33),
         )
 
     async def _uniqSortArray(self, todoprops, layers):
@@ -820,6 +821,9 @@ class ModelRev:
         await self._normPropValu(layers, 'transport:air:craft:model')
         await self._normPropValu(layers, 'transport:sea:vessel:model')
 
+    async def revModel_0_2_33(self, layers):
+        await self._propToForm(layers, 'transport:sea:vessel:name', 'entity:name')
+
     async def runStorm(self, text, opts=None):
         '''
         Run storm code in a schedcoro and log the output messages.

synapse/lib/nexus.py

@@ -413,7 +413,7 @@ class NexsRoot(s_base.Base):
 
         return await func(nexus, *args, **kwargs)
 
-    async def iter(self, offs: int, tellready=False) -> AsyncIterator[Any]:
+    async def iter(self, offs: int, tellready=False, wait=True) -> AsyncIterator[Any]:
         '''
         Returns an iterator of change entries in the log
         '''
@@ -434,6 +434,9 @@ class NexsRoot(s_base.Base):
         if tellready:
             yield None
 
+        if not wait:
+            return
+
         async with self.getChangeDist(maxoffs) as dist:
             async for item in dist:
                 if self.isfini:

synapse/lib/reflect.py

@@ -87,17 +87,16 @@ def getShareInfo(item):
         if not callable(attr):
             continue
 
+        meths[name] = meth = {}
+
         # We know we can cleanly unwrap these functions
         # for asyncgenerator inspection.
         wrapped = getattr(attr, '__syn_wrapped__', None)
         if wrapped in unwraps:
-            real = inspect.unwrap(attr)
-            if inspect.isasyncgenfunction(real):
-                meths[name] = {'genr': True}
-                continue
+            attr = inspect.unwrap(attr)
 
         if inspect.isasyncgenfunction(attr):
-            meths[name] = {'genr': True}
+            meth['genr'] = True
 
     try:
         setattr(item, key, info)