synapse 2.195.1__py311-none-any.whl → 2.197.0__py311-none-any.whl

synapse/lib/stormlib/aha.py

@@ -1,4 +1,7 @@
+import textwrap
+
 import synapse.exc as s_exc
+import synapse.common as s_common
 import synapse.lib.stormtypes as s_stormtypes
 
 @s_stormtypes.registry.registerLib
@@ -49,6 +52,80 @@ class AhaLib(s_stormtypes.Lib):
          'type': {'type': 'function', '_funcname': '_methAhaList', 'args': (),
                   'returns': {'name': 'Yields', 'type': 'list',
                               'desc': 'The AHA service dictionaries.', }}},
+        {'name': 'callPeerApi', 'desc': '''Call an API on all peers (leader and mirrors) of an AHA service and yield the responses from each.
+
+        Examples:
+            Call getCellInfo on an AHA service::
+
+                $todo = $lib.utils.todo('getCellInfo')
+                for $info in $lib.aha.callPeerApi(cortex..., $todo) {
+                    $lib.print($info)
+                }
+
+            Call getCellInfo on an AHA service, skipping the invoking service::
+
+                $todo = $lib.utils.todo('getCellInfo')
+                for $info in $lib.aha.callPeerApi(cortex..., $todo, skiprun=$lib.cell.getCellInfo().cell.run) {
+                    $lib.print($info)
+                }
+
+            Call method with arguments::
+
+                $todo = $lib.utils.todo(('method', ([1, 2]), ({'foo': 'bar'})))
+                for $info in $lib.aha.callPeerApi(cortex..., $todo) {
+                    $lib.print($info)
+                }
+
+        ''',
+         'type': {'type': 'function', '_funcname': '_methCallPeerApi',
+                  'args': (
+                      {'name': 'svcname', 'type': 'str',
+                       'desc': 'The name of the AHA service to call. It is easiest to use the relative name of a service, ending with "...".', },
+                      {'name': 'todo', 'type': 'list',
+                       'desc': 'The todo tuple (name, args, kwargs).'},
+                      {'name': 'timeout', 'type': 'int', 'default': None,
+                       'desc': 'Optional timeout in seconds.'},
+                      {'name': 'skiprun', 'type': 'str', 'default': None,
+                       'desc': '''Optional run ID argument that allows skipping results from a specific service run ID.
+                                  This is most often used to omit the invoking service from the results, ensuring that only responses from other services are included.
+                        '''},
+                  ),
+                  'returns': {'name': 'yields', 'type': 'list',
+                             'desc': 'Yields the results of the API calls as tuples of (svcname, (ok, info)).', }}},
+        {'name': 'callPeerGenr', 'desc': '''Call a generator API on all peers (leader and mirrors) of an AHA service and yield the responses from each.
+
+        Examples:
+            Call getNexusChanges on an AHA service::
+
+                $todo = $lib.utils.todo('getNexusChanges', (0), wait=$lib.false)
+                for $info in $lib.aha.callPeerGenr(cortex..., $todo) {
+                    $lib.print($info)
+                }
+
+            Call getNexusChanges on an AHA service, skipping the invoking service::
+
+                $todo = $lib.utils.todo('getNexusChanges', (0), wait=$lib.false)
+                for $info in $lib.aha.callPeerGenr(cortex..., $todo, skiprun=$lib.cell.getCellInfo().cell.run) {
+                    $lib.print($info)
+                }
+
+        ''',
+         'type': {'type': 'function', '_funcname': '_methCallPeerGenr',
+                  'args': (
+                      {'name': 'svcname', 'type': 'str',
+                       'desc': 'The name of the AHA service to call. It is easiest to use the relative name of a service, ending with "...".', },
+                      {'name': 'todo', 'type': 'list',
+                       'desc': 'The todo tuple (name, args, kwargs).'},
+                      {'name': 'timeout', 'type': 'int', 'default': None,
+                       'desc': 'Optional timeout in seconds.'},
+                      {'name': 'skiprun', 'type': 'str', 'default': None,
+                       'desc': '''Optional run ID argument that allows skipping results from a specific service run ID.
+                                  This is most often used to omit the invoking service from the results, ensuring that only responses from other services are included.
+                       '''},
+                  ),
+                  'returns': {'name': 'yields', 'type': 'list',
+                             'desc': 'Yields the results of the API calls as tuples containing (svcname, (ok, info)).', }}}
+
     )
     _storm_lib_path = ('aha',)
     def getObjLocals(self):
@@ -56,6 +133,8 @@ class AhaLib(s_stormtypes.Lib):
             'del': self._methAhaDel,
             'get': self._methAhaGet,
             'list': self._methAhaList,
+            'callPeerApi': self._methCallPeerApi,
+            'callPeerGenr': self._methCallPeerGenr,
         }
 
     @s_stormtypes.stormfunc(readonly=True)
@@ -85,6 +164,62 @@ class AhaLib(s_stormtypes.Lib):
         proxy = await self.runt.snap.core.reqAhaProxy()
         return await proxy.getAhaSvc(svcname, filters=filters)
 
+    async def _methCallPeerApi(self, svcname, todo, timeout=None, skiprun=None):
+        '''
+        Call an API on an AHA service.
+
+        Args:
+            svcname (str): The name of the AHA service to call.
+            todo (list): The todo tuple from $lib.utils.todo().
+            timeout (int): Optional timeout in seconds.
+            skiprun (str): Optional run ID argument allows skipping self-enumeration.
+        '''
+        svcname = await s_stormtypes.tostr(svcname)
+        todo = await s_stormtypes.toprim(todo)
+        timeout = await s_stormtypes.toint(timeout, noneok=True)
+        skiprun = await s_stormtypes.tostr(skiprun, noneok=True)
+
+        proxy = await self.runt.snap.core.reqAhaProxy()
+        svc = await proxy.getAhaSvc(svcname)
+        if svc is None:
+            raise s_exc.NoSuchName(mesg=f'No AHA service found for {svcname}')
+
+        svcinfo = svc.get('svcinfo')
+        svciden = svcinfo.get('iden')
+        if svciden is None:
+            raise s_exc.NoSuchName(mesg=f'Service {svcname} has no iden')
+
+        async for svcname, (ok, info) in proxy.callAhaPeerApi(svciden, todo, timeout=timeout, skiprun=skiprun):
+            yield (svcname, (ok, info))
+
+    async def _methCallPeerGenr(self, svcname, todo, timeout=None, skiprun=None):
+        '''
+        Call a generator API on an AHA service.
+
+        Args:
+            svcname (str): The name of the AHA service to call.
+            todo (list): The todo tuple from $lib.utils.todo().
+            timeout (int): Optional timeout in seconds.
+            skiprun (str): Optional run ID argument allows skipping self-enumeration.
+        '''
+        svcname = await s_stormtypes.tostr(svcname)
+        todo = await s_stormtypes.toprim(todo)
+        timeout = await s_stormtypes.toint(timeout, noneok=True)
+        skiprun = await s_stormtypes.tostr(skiprun, noneok=True)
+
+        proxy = await self.runt.snap.core.reqAhaProxy()
+        svc = await proxy.getAhaSvc(svcname)
+        if svc is None:
+            raise s_exc.NoSuchName(mesg=f'No AHA service found for {svcname}')
+
+        svcinfo = svc.get('svcinfo')
+        svciden = svcinfo.get('iden')
+        if svciden is None:
+            raise s_exc.NoSuchName(mesg=f'Service {svcname} has no iden')
+
+        async for svcname, (ok, info) in proxy.callAhaPeerGenr(svciden, todo, timeout=timeout, skiprun=skiprun):
+            yield (svcname, (ok, info))
+
 @s_stormtypes.registry.registerLib
 class AhaPoolLib(s_stormtypes.Lib):
     '''
@@ -531,5 +666,220 @@ The ready column indicates that a service has entered into the realtime change w
             }
         }
         '''
-    }
+    },
+    {
+        'name': 'aha.svc.mirror',
+        'descr': textwrap.dedent('''\
+            Query the AHA services and their mirror relationships.
+
+            Note: non-mirror services are not displayed.
+        '''),
+        'cmdargs': (
+            ('--timeout', {'help': 'The timeout in seconds for individual service API calls.',
+                           'default': 10, 'type': 'int'}),
+            ('--wait', {'help': 'Whether to wait for the mirrors to sync.',
+                        'action': 'store_true'}),
+        ),
+        'storm': '''
+        init {
+            $conf = ({
+                "columns": [
+                    {"name": "name", "width": 40},
+                    {"name": "role", "width": 9},
+                    {"name": "online", "width": 7},
+                    {"name": "ready", "width": 6},
+                    {"name": "host", "width": 16},
+                    {"name": "port", "width": 8},
+                    {"name": "version", "width": 12},
+                    {"name": "nexus idx", "width": 10},
+                ],
+                "separators": {
+                    "row:outline": false,
+                    "column:outline": false,
+                    "header:row": "#",
+                    "data:row": "",
+                    "column": "",
+                },
+            })
+            $printer = $lib.tabular.printer($conf)
+            $timeout = $cmdopts.timeout
+            $wait = $cmdopts.wait
+        }
+
+        function get_cell_infos(vname, timeout) {
+            $cell_infos = ({})
+            $todo = $lib.utils.todo('getCellInfo')
+            for $info in $lib.aha.callPeerApi($vname, $todo, timeout=$timeout) {
+                $svcname = $info.0
+                ($ok, $info) = $info.1
+                if $ok {
+                    $cell_infos.$svcname = $info
+                }
+            }
+            return($cell_infos)
+        }
+
+        function build_status_list(members, cell_infos) {
+            $group_status = ()
+            for $svc in $members {
+                $svcinfo = $svc.svcinfo
+                $svcname = $svc.name
+                $status = ({
+                    'name': $svcname,
+                    'role': '<unknown>',
+                    'online': $lib.dict.has($svcinfo, 'online'),
+                    'ready': $svcinfo.ready,
+                    'host': $svcinfo.urlinfo.host,
+                    'port': $svcinfo.urlinfo.port,
+                    'version': '<unknown>',
+                    'nexs_indx': (0)
+                })
+                if ($cell_infos.$svcname) {
+                    $info = $cell_infos.$svcname
+                    $cell_info = $info.cell
+                    $status.nexs_indx = $cell_info.nexsindx
+                    if ($cell_info.uplink) {
+                        $status.role = 'follower'
+                    } else {
+                        $status.role = 'leader'
+                    }
+                    $status.version = $info.synapse.verstring
+                }
+                $group_status.append($status)
+            }
+            return($group_status)
+        }
+
+        function check_sync_status(group_status) {
+            $indices = $lib.set()
+            $known_count = (0)
+            for $status in $group_status {
+                $indices.add($status.nexs_indx)
+                $known_count = ($known_count + (1))
+            }
+            if ($lib.len($indices) = 1) {
+                if ($known_count = $lib.len($group_status)) {
+                    return(true)
+                }
+            }
+        }
+
+        function output_status(vname, group_status, printer) {
+            $lib.print($printer.header())
+            $lib.print($vname)
+            for $status in $group_status {
+                if ($status.nexs_indx = 0) {
+                    $status.nexs_indx = '<unknown>'
+                }
+                $row = (
+                    $status.name,
+                    $status.role,
+                    $status.online,
+                    $status.ready,
+                    $status.host,
+                    $status.port,
+                    $status.version,
+                    $status.nexs_indx
+                )
+                $lib.print($printer.row($row))
+            }
+        }
+
+        $virtual_services = ({})
+        $member_servers = ({})
+
+        for $svc in $lib.aha.list() {
+            $name = $svc.name
+            $svcinfo = $svc.svcinfo
+            $urlinfo = $svcinfo.urlinfo
+            $hostname = $urlinfo.hostname
+
+            if ($name != $hostname) {
+                $virtual_services.$name = $svc
+            } else {
+                $member_servers.$name = $svc
+            }
+        }
+
+        $mirror_groups = ({})
+        for ($vname, $vsvc) in $virtual_services {
+            $vsvc_info = $vsvc.svcinfo
+            $vsvc_iden = $vsvc_info.iden
+            $vsvc_leader = $vsvc_info.leader
+            $vsvc_hostname = $vsvc_info.urlinfo.hostname
+
+            if (not $vsvc_iden or not $vsvc_hostname or not $vsvc_leader) {
+                continue
+            }
+
+            $primary_member = $member_servers.$vsvc_hostname
+            if (not $primary_member) {
+                continue
+            }
+
+            $members = ([$primary_member])
+            for ($mname, $msvc) in $member_servers {
+                if ($mname != $vsvc_hostname) {
+                    $msvc_info = $msvc.svcinfo
+                    if ($msvc_info.iden = $vsvc_iden and $msvc_info.leader = $vsvc_leader) {
+                        $members.append($msvc)
+                    }
+                }
+            }
+
+            if ($lib.len($members) > 1) {
+                $mirror_groups.$vname = $members
+            }
+        }
+
+        for ($vname, $members) in $mirror_groups {
+            $cell_infos = $get_cell_infos($vname, $timeout)
+            $group_status = $build_status_list($members, $cell_infos)
+            $lib.print('Service Mirror Groups:')
+            $output_status($vname, $group_status, $printer)
+
+            if $check_sync_status($group_status) {
+                $lib.print('Group Status: In Sync')
+            } else {
+                $lib.print(`Group Status: Out of Sync`)
+                if $wait {
+                    $leader_nexs = (0)
+                    for $status in $group_status {
+                        if (($status.role = 'leader') and ($status.nexs_indx > 0)) {
+                            $leader_nexs = $status.nexs_indx
+                        }
+                    }
+                    if ($leader_nexs > 0) {
+                        while (true) {
+                            $responses = ()
+                            $todo = $lib.utils.todo(waitNexsOffs, ($leader_nexs - 1), timeout=$timeout)
+                            for $info in $lib.aha.callPeerApi($vname, $todo, timeout=$timeout) {
+                                $svcname = $info.0
+                                ($ok, $info) = $info.1
+                                if ($ok and $info) {
+                                    $responses.append(($svcname, $info))
+                                }
+                            }
+
+                            if ($lib.len($responses) = $lib.len($members)) {
+                                $cell_infos = $get_cell_infos($vname, $timeout)
+                                $group_status = $build_status_list($members, $cell_infos)
+
+                                $lib.print('')
+                                $lib.print('Updated status:')
+                                $output_status($vname, $group_status, $printer)
+
+                                if $check_sync_status($group_status) {
+                                    $lib.print('Group Status: In Sync')
+                                    break
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            $lib.print('')
+        }
+        '''
+    },
 )

synapse/lib/stormlib/graph.py

@@ -5,67 +5,6 @@ import synapse.lib.msgpack as s_msgpack
 import synapse.lib.schemas as s_schemas
 import synapse.lib.stormtypes as s_stormtypes
 
-gdefSchema = {
-    'type': 'object',
-    'properties': {
-        'iden': {'type': 'string', 'pattern': s_config.re_iden},
-        'name': {'type': 'string', 'minLength': 1},
-        'desc': {'type': 'string', 'default': ''},
-        'scope': {'type': 'string', 'enum': ['user', 'power-up']},
-        'creator': {'type': 'string', 'pattern': s_config.re_iden},
-        'power-up': {'type': 'string', 'minLength': 1},
-        'maxsize': {'type': 'number', 'minimum': 0},
-        'existing': {'type': 'array', 'items': {'type': 'string'}},
-        'created': {'type': 'number'},
-        'updated': {'type': 'number'},
-        'refs': {'type': 'boolean', 'default': False},
-        'edges': {'type': 'boolean', 'default': True},
-        'edgelimit': {'type': 'number', 'default': 3000},
-        'degrees': {'type': ['integer', 'null'], 'minimum': 0},
-        'filterinput': {'type': 'boolean', 'default': True},
-        'yieldfiltered': {'type': 'boolean', 'default': False},
-        'filters': {
-            'type': ['array', 'null'],
-            'items': {'type': 'string'}
-        },
-        'pivots': {
-            'type': ['array', 'null'],
-            'items': {'type': 'string'}
-        },
-        'forms': {
-            'type': 'object',
-            'patternProperties': {
-                '^.*$': {
-                    'type': 'object',
-                    'properties': {
-                        'filters': {
-                            'type': ['array', 'null'],
-                            'items': {'type': 'string'}
-                        },
-                        'pivots': {
-                            'type': ['array', 'null'],
-                            'items': {'type': 'string'}
-                        }
-                    },
-                    'additionalProperties': False,
-                }
-            }
-        },
-        'permissions': s_msgpack.deepcopy(s_schemas.easyPermSchema)
-    },
-    'additionalProperties': False,
-    'required': ['iden', 'name', 'scope'],
-    'allOf': [
-        {
-            'if': {'properties': {'scope': {'const': 'power-up'}}},
-            'then': {'required': ['power-up']},
-            'else': {'required': ['creator']},
-        }
-    ]
-}
-
-reqValidGdef = s_config.getJsValidator(gdefSchema)
-
 USER_EDITABLE = {
     'desc',
     'name',

synapse/lib/stormlib/index.py

@@ -0,0 +1,52 @@
+import synapse.lib.stormtypes as s_stormtypes
+
+stormcmds = (
+    {
+        'name': 'index.count.prop',
+        'descr': '''
+            Display the number of properties or property values in the view.
+
+            Examples:
+
+                // Display the number of file:path:ext properties in the view.
+                index.count.prop file:path:ext
+
+                // Display the number of file:path:ext properties with the value
+                // "exe" in the view.
+                index.count.prop file:path:ext --value exe
+
+        ''',
+        'cmdargs': (
+            ('prop', {'help': 'The name of the form or property to count.'}),
+            ('--value', {'help': 'The specific value to count instances of.',
+                         'default': s_stormtypes.undef}),
+        ),
+        'storm': '''
+
+        $conf = (
+            {"columns": [
+                {"name": "Count", "width": 12},
+                {"name": "Layer Iden", "width": 32},
+                {"name": "Layer Name"},
+            ],
+            "separators": {
+                'data:row': ''
+            },
+        })
+        $printer = $lib.tabular.printer($conf)
+
+        $lib.print($printer.header())
+
+        $total = (0)
+        for $layer in $lib.view.get().layers {
+
+            $count = $layer.getPropCount($cmdopts.prop, valu=$cmdopts.value)
+
+            $total = ($total + $count)
+
+            $lib.print($printer.row(($count, $layer.iden, $layer.name)))
+        }
+        $lib.print(`Total: {$total}`)
+        ''',
+    },
+)

synapse/lib/stormlib/utils.py

@@ -0,0 +1,37 @@
+import synapse.lib.stormtypes as s_stormtypes
+
+@s_stormtypes.registry.registerLib
+class LibUtils(s_stormtypes.Lib):
+    '''
+    A Storm library for working with utility functions.
+    '''
+    _storm_locals = (
+        {'name': 'todo',
+         'desc': '''
+            Create a todo tuple of (name, args, kwargs).
+         ''',
+         'type': {'type': 'function', '_funcname': '_todo',
+                  'args': (
+                      {'name': '_todoname', 'type': 'str',
+                       'desc': 'The todo name.'},
+                      {'name': '*args', 'type': 'any',
+                       'desc': 'Positional arguments for the todo.'},
+                      {'name': '**kwargs', 'type': 'any',
+                       'desc': 'Keyword arguments for the todo.'},
+                  ),
+                  'returns': {'type': 'list', 'desc': 'A todo tuple of (name, args, kwargs).'},
+        }},
+    )
+    _storm_lib_path = ('utils',)
+
+    def getObjLocals(self):
+        return {
+            'todo': self._todo,
+        }
+
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _todo(self, _todoname, *args, **kwargs):
+        _todoname = await s_stormtypes.tostr(_todoname)
+        args = await s_stormtypes.toprim(args)
+        kwargs = await s_stormtypes.toprim(kwargs)
+        return (_todoname, args, kwargs)