synapse 2.176.0__py311-none-any.whl → 2.178.0__py311-none-any.whl

synapse/lib/nexus.py

@@ -91,7 +91,6 @@ class NexsRoot(s_base.Base):
         self.writeholds = set()
 
         self.applytask = None
-        self.applylock = asyncio.Lock()
 
         self.ready = asyncio.Event()
         self.donexslog = self.cell.conf.get('nexslog:en')
@@ -110,9 +109,7 @@ class NexsRoot(s_base.Base):
 
         logpath = s_common.genpath(self.dirn, 'slabs', 'nexuslog')
 
-        self.map_async = self.cell.conf.get('nexslog:async')
-        self.nexsslab = await s_lmdbslab.Slab.anit(path, map_async=self.map_async)
-        self.nexsslab.addResizeCallback(cell.checkFreeSpace)
+        self.nexsslab = await cell._initSlabFile(path)
 
         self.nexshot = await self.nexsslab.getHotCount('nexs:indx')
 
@@ -126,8 +123,7 @@ class NexsRoot(s_base.Base):
         elif vers != 2:
             raise s_exc.BadStorageVersion(mesg=f'Got nexus log version {vers}.  Expected 2.  Accidental downgrade?')
 
-        slabopts = {'map_async': self.map_async}
-        self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, slabopts=slabopts, cell=cell)
+        self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, cell=cell)
 
         # just in case were previously configured differently
         logindx = self.nexslog.index()
@@ -147,6 +143,9 @@ class NexsRoot(s_base.Base):
 
         self.onfini(fini)
 
+    def getNexsKids(self):
+        return list(self._nexskids.values())
+
     async def _migrateV1toV2(self, nexspath, logpath):
         '''
         Close the slab, move it to the new multislab location, then copy out the nexshot
@@ -195,8 +194,7 @@ class NexsRoot(s_base.Base):
 
         # Open a fresh slab where the old one used to be
         logger.warning(f'Re-opening fresh nexslog slab at {nexspath} for nexshot')
-        self.nexsslab = await s_lmdbslab.Slab.anit(nexspath, map_async=self.map_async)
-        self.nexsslab.addResizeCallback(self.cell.checkFreeSpace)
+        self.nexsslab = await self.cell._initSlabFile(nexspath)
 
         self.nexshot = await self.nexsslab.getHotCount('nexs:indx')
 
@@ -230,7 +228,7 @@ class NexsRoot(s_base.Base):
 
     async def enNexsLog(self):
 
-        async with self.applylock:
+        async with self.cell.nexslock:
 
             if self.donexslog:
                 return
@@ -309,7 +307,6 @@ class NexsRoot(s_base.Base):
         If I'm not a follower, mutate, otherwise, ask the leader to make the change and wait for the follower loop
         to hand me the result through a future.
         '''
-
         # pick up a reference to avoid race when we eventually can promote
         client = self.client
 
@@ -344,7 +341,7 @@ class NexsRoot(s_base.Base):
         if meta is None:
             meta = {}
 
-        async with self.applylock:
+        async with self.cell.nexslock:
             self.reqNotReadOnly()
             # Keep a reference to the shielded task to ensure it isn't GC'd
             self.applytask = asyncio.create_task(self._eat((nexsiden, event, args, kwargs, meta)))
@@ -577,6 +574,9 @@ class NexsRoot(s_base.Base):
                         if respfutu is not None:
                             respfutu.set_result(retn)
 
+            except s_exc.LinkShutDown:
+                logger.warning(f'mirror loop: leader closed the connection.')
+
             except Exception as exc:  # pragma: no cover
                 logger.exception(f'error in mirror loop: {exc}')
 
@@ -634,9 +634,21 @@ class Pusher(s_base.Base, metaclass=RegMethType):
             assert prev is not None, f'Failed removing {self.nexsiden}'
 
         self.onfini(onfini)
-
         self.nexsroot = nexsroot
 
+    async def modNexsRoot(self, ctor):
+
+        kids = [self]
+        if self.nexsroot is not None:
+            kids = self.nexsroot.getNexsKids()
+            await self.nexsroot.fini()
+
+        nexsroot = await ctor()
+
+        [kid.setNexsRoot(nexsroot) for kid in kids]
+
+        await nexsroot.startup()
+
     @classmethod
     def onPush(cls, event: str, passitem=False) -> Callable:
         '''

synapse/lib/schemas.py

@@ -288,6 +288,93 @@ _stormPoolOptsSchema = {
 }
 reqValidStormPoolOpts = s_config.getJsValidator(_stormPoolOptsSchema)
 
+_authRulesSchema = {
+    'type': 'array',
+    'items': {
+        'type': 'array',
+        'items': [
+            {'type': 'boolean'},
+            {'type': 'array', 'items': {'type': 'string'}},
+        ],
+        'minItems': 2,
+        'maxItems': 2,
+    }
+}
+reqValidRules = s_config.getJsValidator(_authRulesSchema)
+
+_passwdPolicySchema = {
+    'type': 'object',
+    'properties': {
+        'complexity': {
+            'type': ['object', 'null'],
+            'properties': {
+                'length': {
+                    'type': ['number', 'null'],
+                    'minimum': 1,
+                    'description': 'Minimum password character length.',
+                },
+                'sequences': {
+                    'type': ['number', 'null'],
+                    'minimum': 2,
+                    'description': 'Maximum sequence length in a password. Sequences can be letters or number, forward or reverse.',
+                },
+                'upper:count': {
+                    'type': ['number', 'null'],
+                    'description': 'The minimum number of uppercase characters required in password.',
+                },
+                'upper:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid uppercase characters.',
+                },
+                'lower:count': {
+                    'type': ['number', 'null'],
+                    'minimum': 0,
+                    'description': 'The minimum number of lowercase characters required in password.',
+                },
+                'lower:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid lowercase characters.',
+                },
+                'special:count': {
+                    'type': ['number', 'null'],
+                    'minimum': 0,
+                    'description': 'The minimum number of special characters required in password.',
+                },
+                'special:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid special characters.',
+                },
+                'number:count': {
+                    'type': ['number', 'null'],
+                    'minimum': 0,
+                    'description': 'The minimum number of digit characters required in password.',
+                },
+                'number:valid': {
+                    'type': ['string', 'null'],
+                    'minLength': 1,
+                    'description': 'All valid digit characters.',
+                },
+            },
+            'additionalProperties': False,
+        },
+        'attempts': {
+            'type': ['number', 'null'],
+            'minimum': 1,
+            'description': 'Maximum number of incorrect attempts before locking user account.',
+        },
+        'previous': {
+            'type': ['number', 'null'],
+            'minimum': 1,
+            'description': 'Number of previous passwords to disallow.',
+        },
+    },
+    'additionalProperties': False,
+}
+reqValidPasswdPolicy = s_config.getJsValidator(_passwdPolicySchema)
+
 # These types are order sensitive
 _changelogTypes = {'migration': 'Automatic Migrations',
                    'model': 'Model Changes',
@@ -318,3 +405,52 @@ _changelogSchema = {
     'required': ['type', 'desc']
 }
 _reqChanglogSchema = s_config.getJsValidator(_changelogSchema)
+
+tabularConfSchema = {
+    'type': 'object',
+    'properties': {
+        'separators': {
+            'type': 'object',
+            'properties': {
+                'row:outline': {'type': 'boolean', 'default': False,
+                                'description': 'Add the row separator before the header data and after each row.'},
+                'column:outline': {'type': 'boolean', 'default': False,
+                                   'description': 'Add the column separator to the beginning and end of each row.'},
+                'header:row': {'type': 'string', 'default': '=',
+                               'description': 'The string to use to create a separator row when printing the header.'},
+                'data:row': {'type': 'string', 'default': '-',
+                             'description': 'The string to use to create a separator row when printing data rows.'},
+                'column': {'type': 'string', 'default': '|',
+                           'description': 'The string to use to separate columns.'},
+            },
+            'additionalProperties': False,
+        },
+        'columns': {
+            'type': 'array',
+            'items': {
+                'type': 'object',
+                'properties': {
+                    'name': {'type': 'string',
+                             'description': 'The column name which will be used in the header row.'},
+                    'width': {'type': 'number', 'default': None, 'exclusiveMinimum': 0,
+                              'description': 'If not provided each cell will expand to fit the data.'},
+                    'justify': {'type': 'string', 'default': 'left', 'enum': ['left', 'center', 'right'],
+                                'description': 'Justification for the header titles and data rows.'},
+                    'overflow': {'type': 'string', 'default': 'trim', 'enum': ['wrap', 'trim'],
+                                 'description': 'For text exceeding the width, '
+                                                'either wrap text in multiple lines or trim and append "...".'},
+                    'newlines': {'type': 'string', 'default': 'replace', 'enum': ['replace', 'split'],
+                                 'description': 'Replace newlines with a space or split into multiple lines.'
+                                               'Split is only applied if width is undefined.'},
+                },
+                'required': ['name'],
+                'minItems': 1,
+                'additionalProperties': False,
+            },
+        },
+    },
+    'required': ['columns'],
+    'additionalProperties': False,
+}
+
+reqValidTabularConf = s_config.getJsValidator(tabularConfSchema)

synapse/lib/storm.py

@@ -12,6 +12,7 @@ import synapse.telepath as s_telepath
 import synapse.datamodel as s_datamodel
 
 import synapse.lib.ast as s_ast
+import synapse.lib.auth as s_auth
 import synapse.lib.base as s_base
 import synapse.lib.chop as s_chop
 import synapse.lib.coro as s_coro
@@ -28,7 +29,6 @@ import synapse.lib.msgpack as s_msgpack
 import synapse.lib.spooled as s_spooled
 import synapse.lib.version as s_version
 import synapse.lib.hashitem as s_hashitem
-import synapse.lib.hiveauth as s_hiveauth
 import synapse.lib.stormctrl as s_stormctrl
 import synapse.lib.stormtypes as s_stormtypes
 
@@ -1066,7 +1066,7 @@ stormcmds = (
                 $lib.view.del($view.iden)
                 $lib.layer.del($layriden)
             } else {
-                $view.wipeLayer()
+                $view.swapLayer()
             }
             $lib.print("View merged: {iden}", iden=$cmdopts.iden)
         ''',
@@ -1831,7 +1831,7 @@ class Runtime(s_base.Base):
 
     '''
 
-    _admin_reason = s_hiveauth._allowedReason(True, isadmin=True)
+    _admin_reason = s_auth._allowedReason(True, isadmin=True)
     async def __anit__(self, query, snap, opts=None, user=None, root=None):
 
         await s_base.Base.__anit__(self)
@@ -3728,6 +3728,8 @@ class MergeCmd(Cmd):
         pars = Cmd.getArgParser(self)
         pars.add_argument('--apply', default=False, action='store_true',
                           help='Execute the merge changes.')
+        pars.add_argument('--wipe', default=False, action='store_true',
+                          help='Replace the top layer in the view with a fresh layer.')
         pars.add_argument('--no-tags', default=False, action='store_true',
                           help='Do not merge tags/tagprops or syn:tag nodes.')
         pars.add_argument('--only-tags', default=False, action='store_true',
@@ -3801,13 +3803,15 @@ class MergeCmd(Cmd):
         layr1 = runt.snap.view.layers[1].iden
 
         if not allows['forms'] and sode.get('valu') is not None:
-            runt.confirm(('node', 'del', node.form.name), gateiden=layr0)
+            if not self.opts.wipe:
+                runt.confirm(('node', 'del', node.form.name), gateiden=layr0)
             runt.confirm(('node', 'add', node.form.name), gateiden=layr1)
 
         if not allows['props']:
             for name in sode.get('props', {}).keys():
                 prop = node.form.prop(name)
-                runt.confirmPropDel(prop, layriden=layr0)
+                if not self.opts.wipe:
+                    runt.confirmPropDel(prop, layriden=layr0)
                 runt.confirmPropSet(prop, layriden=layr1)
 
         if not allows['tags']:
@@ -3818,7 +3822,8 @@ class MergeCmd(Cmd):
                 if valu != (None, None):
                     tagadds.append(tag)
                     tagperm = tuple(tag.split('.'))
-                    runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
+                    if not self.opts.wipe:
+                        runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
                     runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
                 else:
                     tags.append((len(tag), tag))
@@ -3830,22 +3835,26 @@ class MergeCmd(Cmd):
 
                 tagadds.append(tag)
                 tagperm = tuple(tag.split('.'))
-                runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
+                if not self.opts.wipe:
+                    runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
                 runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
 
             for tag in sode.get('tagprops', {}).keys():
                 tagperm = tuple(tag.split('.'))
-                runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
+                if not self.opts.wipe:
+                    runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0)
                 runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1)
 
         if not allows['ndata']:
             async for name in runt.snap.view.layers[0].iterNodeDataKeys(node.buid):
-                runt.confirm(('node', 'data', 'pop', name), gateiden=layr0)
+                if not self.opts.wipe:
+                    runt.confirm(('node', 'data', 'pop', name), gateiden=layr0)
                 runt.confirm(('node', 'data', 'set', name), gateiden=layr1)
 
         if not allows['edges']:
             async for verb in runt.snap.view.layers[0].iterNodeEdgeVerbsN1(node.buid):
-                runt.confirm(('node', 'edge', 'del', verb), gateiden=layr0)
+                if not self.opts.wipe:
+                    runt.confirm(('node', 'edge', 'del', verb), gateiden=layr0)
                 runt.confirm(('node', 'edge', 'add', verb), gateiden=layr1)
 
     async def execStormCmd(self, runt, genr):
@@ -3854,6 +3863,11 @@ class MergeCmd(Cmd):
             mesg = 'You may only merge nodes in forked views'
             raise s_exc.CantMergeView(mesg=mesg)
 
+        if self.opts.wipe:
+            mesg = 'merge --wipe requires view admin'
+            runt.reqAdmin(gateiden=runt.snap.view.iden, mesg=mesg)
+            runt.confirm(('layer', 'del'), gateiden=runt.snap.view.layers[0].iden)
+
         notags = self.opts.no_tags
         onlytags = self.opts.only_tags
         doapply = self.opts.apply
@@ -3867,18 +3881,27 @@ class MergeCmd(Cmd):
         doperms = doapply and not (runt.isAdmin(gateiden=layr0.iden) and runt.isAdmin(gateiden=layr1.iden))
 
         if doperms:
-            allows = {
-                'forms': runt.user.allowed(('node', 'del'), gateiden=layr0.iden, deepdeny=True) and
-                         runt.user.allowed(('node', 'add'), gateiden=layr1.iden, deepdeny=True),
-                'props': runt.user.allowed(('node', 'prop', 'del'), gateiden=layr0.iden, deepdeny=True) and
-                         runt.user.allowed(('node', 'prop', 'set'), gateiden=layr1.iden, deepdeny=True),
-                'tags': runt.user.allowed(('node', 'tag', 'del'), gateiden=layr0.iden, deepdeny=True) and
-                        runt.user.allowed(('node', 'tag', 'add'), gateiden=layr1.iden, deepdeny=True),
-                'ndata': runt.user.allowed(('node', 'data', 'pop'), gateiden=layr0.iden, deepdeny=True) and
-                         runt.user.allowed(('node', 'data', 'set'), gateiden=layr1.iden, deepdeny=True),
-                'edges': runt.user.allowed(('node', 'edge', 'del'), gateiden=layr0.iden, deepdeny=True) and
-                         runt.user.allowed(('node', 'edge', 'add'), gateiden=layr1.iden, deepdeny=True),
-            }
+            if not self.opts.wipe:
+                allows = {
+                    'forms': runt.user.allowed(('node', 'del'), gateiden=layr0.iden, deepdeny=True) and
+                             runt.user.allowed(('node', 'add'), gateiden=layr1.iden, deepdeny=True),
+                    'props': runt.user.allowed(('node', 'prop', 'del'), gateiden=layr0.iden, deepdeny=True) and
+                             runt.user.allowed(('node', 'prop', 'set'), gateiden=layr1.iden, deepdeny=True),
+                    'tags': runt.user.allowed(('node', 'tag', 'del'), gateiden=layr0.iden, deepdeny=True) and
+                            runt.user.allowed(('node', 'tag', 'add'), gateiden=layr1.iden, deepdeny=True),
+                    'ndata': runt.user.allowed(('node', 'data', 'pop'), gateiden=layr0.iden, deepdeny=True) and
+                             runt.user.allowed(('node', 'data', 'set'), gateiden=layr1.iden, deepdeny=True),
+                    'edges': runt.user.allowed(('node', 'edge', 'del'), gateiden=layr0.iden, deepdeny=True) and
+                             runt.user.allowed(('node', 'edge', 'add'), gateiden=layr1.iden, deepdeny=True),
+                }
+            else:
+                allows = {
+                    'forms': runt.user.allowed(('node', 'add'), gateiden=layr1.iden, deepdeny=True),
+                    'props': runt.user.allowed(('node', 'prop', 'set'), gateiden=layr1.iden, deepdeny=True),
+                    'tags': runt.user.allowed(('node', 'tag', 'add'), gateiden=layr1.iden, deepdeny=True),
+                    'ndata': runt.user.allowed(('node', 'data', 'set'), gateiden=layr1.iden, deepdeny=True),
+                    'edges': runt.user.allowed(('node', 'edge', 'add'), gateiden=layr1.iden, deepdeny=True),
+                }
 
             doperms = not all(allows.values())
 
@@ -3966,7 +3989,8 @@ class MergeCmd(Cmd):
                             if name == '.created':
                                 if doapply:
                                     protonode.props['.created'] = valu
-                                    subs.append((s_layer.EDIT_PROP_DEL, (name, valu, stortype), ()))
+                                    if not self.opts.wipe:
+                                        subs.append((s_layer.EDIT_PROP_DEL, (name, valu, stortype), ()))
                                 continue
 
                             isset = False
@@ -3990,7 +4014,8 @@ class MergeCmd(Cmd):
                             await runt.printf(f'{nodeiden} {form}:{name} = {valurepr}')
                         else:
                             await protonode.set(name, valu)
-                            subs.append((s_layer.EDIT_PROP_DEL, (name, valu, stortype), ()))
+                            if not self.opts.wipe:
+                                subs.append((s_layer.EDIT_PROP_DEL, (name, valu, stortype), ()))
 
                 if doapply and protonode is None:
                     protonode = await editor.addNode(form, node.ndef[1], norminfo={})
@@ -4009,7 +4034,8 @@ class MergeCmd(Cmd):
                             await runt.printf(f'{nodeiden} {form}#{tag}{valurepr}')
                         else:
                             await protonode.addTag(tag, valu)
-                            subs.append((s_layer.EDIT_TAG_DEL, (tag, valu), ()))
+                            if not self.opts.wipe:
+                                subs.append((s_layer.EDIT_TAG_DEL, (tag, valu), ()))
 
                     for tag, tagdict in sode.get('tagprops', {}).items():
 
@@ -4022,7 +4048,8 @@ class MergeCmd(Cmd):
                                 await runt.printf(f'{nodeiden} {form}#{tag}:{prop} = {valurepr}')
                             else:
                                 await protonode.setTagProp(tag, prop, valu)
-                                subs.append((s_layer.EDIT_TAGPROP_DEL, (tag, prop, valu, stortype), ()))
+                                if not self.opts.wipe:
+                                    subs.append((s_layer.EDIT_TAGPROP_DEL, (tag, prop, valu, stortype), ()))
 
                 if not onlytags or form == 'syn:tag':
 
@@ -4032,7 +4059,8 @@ class MergeCmd(Cmd):
                             await runt.printf(f'{nodeiden} {form} DATA {name} = {valurepr}')
                         else:
                             await protonode.setData(name, valu)
-                            subs.append((s_layer.EDIT_NODEDATA_DEL, (name, valu), ()))
+                            if not self.opts.wipe:
+                                subs.append((s_layer.EDIT_NODEDATA_DEL, (name, valu), ()))
 
                     async for edge in s_coro.pause(layr0.iterNodeEdgesN1(node.buid)):
                         name, dest = edge
@@ -4040,9 +4068,10 @@ class MergeCmd(Cmd):
                             await runt.printf(f'{nodeiden} {form} +({name})> {dest}')
                         else:
                             await protonode.addEdge(name, dest)
-                            subs.append((s_layer.EDIT_EDGE_DEL, edge, ()))
+                            if not self.opts.wipe:
+                                subs.append((s_layer.EDIT_EDGE_DEL, edge, ()))
 
-                if delnode:
+                if delnode and not self.opts.wipe:
                     subs.append((s_layer.EDIT_NODE_DEL, valu, ()))
 
                 if doapply:
@@ -4057,6 +4086,9 @@ class MergeCmd(Cmd):
                 runt.snap.clearCachedNode(node.buid)
                 yield await runt.snap.getNodeByBuid(node.buid), path
 
+            if doapply and self.opts.wipe:
+                await runt.snap.view.swapLayer()
+
 class MoveNodesCmd(Cmd):
     '''
     Move storage nodes between layers.

synapse/lib/stormlib/aha.py

@@ -486,7 +486,7 @@ The ready column indicates that a service has entered into the realtime change w
                     if $svcinfo.online {
                         $nexusOffset = $_getNexus($name)
                     } else {
-                        $nexusOffset = 'Service is not online. Will not attempt to retrieve its nexus offset.'
+                        $nexusOffset = '<offline>'
                     }
                 }
                 $name=$name.ljust(45)