synapse 2.186.0__py311-none-any.whl → 2.188.0__py311-none-any.whl

synapse/lib/stormlib/model.py

@@ -2,12 +2,11 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.node as s_node
+import synapse.lib.time as s_time
 import synapse.lib.cache as s_cache
 import synapse.lib.layer as s_layer
 import synapse.lib.stormtypes as s_stormtypes
 
-import synapse.models.infotech as s_infotech
-
 RISK_HASVULN_VULNPROPS = (
     'hardware',
     'host',
@@ -818,8 +817,6 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
             'copyEdges': self._methCopyEdges,
             'copyTags': self._methCopyTags,
             'copyExtProps': self._methCopyExtProps,
-            'liftByPropValuNoNorm': self._methLiftByPropValuNoNorm,
-            'setNodePropValuNoNorm': self._methSetNodePropValuNoNorm,
         }
 
     async def _methCopyData(self, src, dst, overwrite=False):
@@ -876,125 +873,12 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
             proto = editor.loadNode(dst)
             await self.copyExtProps(src, proto)
 
-    async def _methLiftByPropValuNoNorm(self, formname, propname, valu, cmpr='=', reverse=False):
-        '''
-        No storm docs for this on purpose. It is restricted for use during model migrations only.
-        '''
-        formname = await s_stormtypes.tostr(formname)
-        propname = await s_stormtypes.tostr(propname)
-        valu = await s_stormtypes.toprim(valu)
-
-        prop = self.runt.snap.core.model.prop(f'{formname}:{propname}')
-        if prop is None:
-            mesg = f'Could not find prop: {formname}:{propname}'
-            raise s_exc.NoSuchProp(mesg=mesg, formname=formname, propname=propname)
-
-        if not self.runt.snap.core.migration:
-            mesg = '$lib.model.migration.liftByPropValuNoNorm() is restricted to model migrations only.'
-            raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
-
-        stortype = prop.type.stortype
-
-        # Normally we'd call proptype.getStorCmprs() here to get the cmprvals
-        # but getStorCmprs() calls norm() which we're  trying to avoid so build
-        # cmprvals manually here.
-
-        if prop.type.isarray:
-            stortype &= (~s_layer.STOR_FLAG_ARRAY)
-            liftfunc = self.runt.snap.wlyr.liftByPropArray
-        else:
-            liftfunc = self.runt.snap.wlyr.liftByPropValu
-
-        cmprvals = ((cmpr, valu, stortype),)
-
-        layriden = self.runt.snap.wlyr.iden
-        async for _, buid, sode in liftfunc(formname, propname, cmprvals, reverse=reverse):
-            yield await self.runt.snap._joinStorNode(buid, {layriden: sode})
-
-    async def _methSetNodePropValuNoNorm(self, n, propname, valu):
-        '''
-        No storm docs for this on purpose. It is restricted for use during model migrations only.
-        '''
-
-        # NB: I'm sure there are all kinds of edges cases that this function doesn't account for. At the time of it's
-        # creation, this was intended to be used to update array properties with bad it:sec:cpe values in them. It works
-        # for that use case (see model migration 0.2.28). Any additional use of this function should perform heavy
-        # testing.
-
-        if not isinstance(n, s_node.Node):
-            raise s_exc.BadArg(mesg='$lib.model.migration.setNodePropValuNoNorm() argument must be a node.')
-
-        if not self.runt.snap.core.migration:
-            mesg = '$lib.model.migration.setNodePropValuNoNorm() is restricted to model migrations only.'
-            raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
-
-        propname = await s_stormtypes.tostr(propname)
-        valu = await s_stormtypes.toprim(valu)
-
-        async with self.runt.snap.getNodeEditor(n) as proto:
-            await proto.set(propname, valu, norminfo={})
-
-        return n
-
 @s_stormtypes.registry.registerLib
 class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
     '''
     A Storm library for selectively migrating nodes in the current view.
     '''
     _storm_locals = (
-        {'name': 'itSecCpe_2_170_0',
-         'desc': '''
-            Versions of Synapse prior to v2.169.0 did not correctly parse and
-            convert CPE strings from 2.2 -> 2.3 or 2.3 -> 2.2. This migration
-            attempts to re-normalize `it:sec:cpe` nodes that may be fixable.
-
-            NOTE: It is highly recommended to test the `it:sec:cpe` migrations
-            in a fork first and confirm the migration was successful without any
-            issues. Then run the migration in view deporder to migrate the
-            entire cortex. E.g.::
-
-                for $view in $lib.view.list(deporder=$lib.true) {
-                    view.exec $view.iden {
-                        for $n in $lib.layer.get().liftByProp(it:sec:cpe) {
-                            $lib.model.migration.s.itSecCpe_2_170_0($n)
-                        }
-                    }
-                }
-
-            Upon completion of the migration, nodedata will contain a
-            `migration.s.itSecCpe_2_170_0` dict with information about the
-            migration status. This dict may contain the following:
-
-                - `status`: (required str) "success" or "failed"
-                - `reason`: (optional str) if "status" is "failed", this key will
-                  explain why the migration failed.
-                - `valu`: (optional str) if this key is present, it will contain
-                  an updated CPE2.3 string since the primary property cannot be
-                  changed.
-                - `updated`: (optional list[str]) A list of properties that were
-                  updated by the migration.
-
-            Failed or incorrect migrations may be helped by updating the :v2_2
-            property to be a valid CPE2.2 string and then re-running the
-            migration with `force=$lib.true`. If the primary property (CPE2.3)
-            is valid but incorrect, users may update the :v2_2 property and then
-            run the migration with `prefer_v22=$lib.true` to make the migration
-            use the `:v2_2` string instead of the primary property for the
-            migration process.
-         ''',
-         'type': {'type': 'function', '_funcname': '_itSecCpe_2_170_0',
-                  'args': (
-                      {'name': 'n', 'type': 'node', 'desc': 'The it:sec:cpe node to migrate.'},
-                      {'name': 'prefer_v22', 'type': 'bool', 'default': False,
-                       'desc': '''
-                        Try to renormalize using the :v2_2 prop instead of the
-                        primary property. This can be especially useful when the
-                        primary property is a valid but incorrect CPE string.
-                        '''},
-                      {'name': 'force', 'type': 'bool', 'default': False,
-                       'desc': 'Perform fixups even if the primary property and :v2_2 are valid.'},
-                  ),
-                  'returns': {'type': 'boolean', 'desc': 'Boolean indicating if the migration was successful.'}}},
         {'name': 'riskHasVulnToVulnerable', 'desc': '''
             Create a risk:vulnerable node from the provided risk:hasvuln node.
 
@@ -1076,142 +960,9 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
 
     def getObjLocals(self):
         return {
-            'itSecCpe_2_170_0': self._itSecCpe_2_170_0,
-            'itSecCpe_2_170_0_internal': self._itSecCpe_2_170_0_internal,
             'riskHasVulnToVulnerable': self._riskHasVulnToVulnerable,
         }
 
-    async def _itSecCpe_2_170_0(self, n, prefer_v22=False, force=False):
-        info = await self._itSecCpe_2_170_0_internal(n, prefer_v22=prefer_v22, force=force, set_nodedata=True)
-        return info.get('status') == 'success'
-
-    async def _itSecCpe_2_170_0_internal(self, n, prefer_v22=False, force=False, set_nodedata=False):
-
-        if not isinstance(n, s_node.Node):
-            raise s_exc.BadArg(mesg='$lib.model.migration.s.itSecCpe_2_170_0() argument must be a node.')
-
-        if n.form.name != 'it:sec:cpe':
-            raise s_exc.BadArg(f'itSecCpeFix only accepts it:sec:cpe nodes, not {n.form.name}')
-
-        prefer_v22 = await s_stormtypes.tobool(prefer_v22)
-        force = await s_stormtypes.tobool(force)
-
-        layr = self.runt.snap.wlyr
-        # We only need to check :v2_2 since that's the only property that's
-        # writable. Everthing else is readonly. And we can do it here once
-        # instead of in the loop below which will cause a perf hit.
-        self.runt.confirmPropSet(n.form.prop('v2_2'), layriden=layr.iden)
-
-        curv = n.repr()
-        reprvalu = f'it:sec:cpe={curv}'
-
-        nodedata = await n.getData('migration.s.itSecCpe_2_170_0', {})
-        if nodedata.get('status') == 'success' and not force:
-            if self.runt.debug:
-                mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node already migrated.'
-                await self.runt.printf(mesg)
-            return nodedata
-
-        modl = self.runt.model.type('it:sec:cpe')
-
-        valu23 = None
-        valu22 = None
-
-        # Check the primary property for validity.
-        cpe23 = s_infotech.cpe23_regex.match(curv)
-        if cpe23 is not None and cpe23.group() == curv:
-            valu23 = curv
-
-        # Check the v2_2 property for validity.
-        v2_2 = n.props.get('v2_2')
-        if v2_2 is not None:
-            rgx = s_infotech.cpe22_regex.match(v2_2)
-            if rgx is not None and rgx.group() == v2_2:
-                valu22 = v2_2
-
-        async with self.runt.snap.getNodeEditor(n) as proto:
-
-            # If both values are populated, this node is valid
-            if valu23 is not None and valu22 is not None and not force:
-                if self.runt.debug:
-                    mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node is valid, no migration necessary.'
-                    await self.runt.printf(mesg)
-
-                nodedata = {'status': 'success'}
-                if set_nodedata:
-                    await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
-
-                return nodedata
-
-            if valu23 is None and valu22 is None:
-                reason = 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.'
-                # Invalid 2.3 string and no/invalid v2_2 prop. Nothing
-                # we can do here so log, mark, and go around.
-                mesg = f'itSecCpe_2_170_0({reprvalu}): {reason}'
-                await self.runt.warn(mesg)
-
-                nodedata = {
-                    'status': 'failed',
-                    'reason': reason,
-                }
-                if set_nodedata:
-                    await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
-
-                return nodedata
-
-            if prefer_v22:
-                valu = valu22 or valu23
-            else:
-                valu = valu23 or valu22
-
-            # Re-normalize the data from the 2.3 or 2.2 string, whichever was valid.
-            norm, info = modl.norm(valu)
-            subs = info.get('subs')
-
-            nodedata = {'status': 'success'}
-
-            if norm != curv:
-                # The re-normed value is not the same as the current value.
-                # Since we can't change the primary property, store the
-                # updated value in nodedata.
-                if self.runt.debug:
-                    mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Stored updated primary property value to nodedata: {curv} -> {norm}.'
-                    await self.runt.printf(mesg)
-
-                nodedata['valu'] = norm
-
-            # Iterate over the existing properties
-            for propname, propcurv in n.props.items():
-                subscurv = subs.get(propname)
-                if subscurv is None:
-                    continue
-
-                if propname == 'v2_2' and isinstance(subscurv, list):
-                    subscurv = s_infotech.zipCpe22(subscurv)
-
-                # Values are the same, go around
-                if propcurv == subscurv:
-                    continue
-
-                nodedata.setdefault('updated', [])
-                nodedata['updated'].append(propname)
-
-                # Update the existing property with the re-normalized property value.
-                await proto.set(propname, subscurv, ignore_ro=True)
-
-            if set_nodedata:
-                await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
-
-            if self.runt.debug:
-                if nodedata.get('updated'):
-                    mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Updated properties: {", ".join(nodedata["updated"])}.'
-                    await self.runt.printf(mesg)
-                else:
-                    mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): No property updates required.'
-                    await self.runt.printf(mesg)
-
-            return nodedata
-
     async def _riskHasVulnToVulnerable(self, n, nodata=False):
 
         nodata = await s_stormtypes.tobool(nodata)
@@ -1269,3 +1020,322 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
                     await self.copyData(n, proto, overwrite=False)
 
         return retidens
+
+@s_stormtypes.registry.registerLib
+class LibModelMigrations_0_2_31(s_stormtypes.Lib):
+    '''
+    A Storm library with helper functions for the 0.2.31 model it:sec:cpe migration.
+    '''
+    _storm_locals = (
+        {'name': 'listNodes', 'desc': 'Yield queued nodes.',
+         'type': {'type': 'function', '_funcname': '_methListNodes',
+                  'args': (
+                      {'name': 'form', 'type': 'form', 'default': None,
+                       'desc': 'Only yield entries matching the specified form.'},
+                      {'name': 'source', 'type': 'str', 'default': None,
+                       'desc': 'Only yield entries that were seen by the specified source.'},
+                      {'name': 'offset', 'type': 'int', 'default': 0,
+                       'desc': 'Skip this many entries.'},
+                      {'name': 'size', 'type': 'int', 'default': None,
+                       'desc': 'Only yield up to this many entries.'},
+                  ),
+                  'returns': {'name': 'Yields', 'type': 'list',
+                              'desc': 'A tuple of (offset, form, valu, sources) values for the specified node.', }}},
+        {'name': 'printNode', 'desc': 'Print detailed queued node information.',
+         'type': {'type': 'function', '_funcname': '_methPrintNode',
+                  'args': (
+                      {'name': 'offset', 'type': 'into', 'desc': 'The offset of the queued node to print.'},
+                  ),
+                  'returns': {'type': 'null'}}},
+        {'name': 'repairNode', 'desc': 'Repair a queued node.',
+         'type': {'type': 'function', '_funcname': '_methRepairNode',
+                  'args': (
+                      {'name': 'offset', 'type': 'str', 'desc': 'The node queue offset to repair.'},
+                      {'name': 'newvalu', 'type': 'any', 'desc': 'The new (corrected) node value.'},
+                      {'name': 'remove', 'type': 'boolean', 'default': False,
+                       'desc': 'Specify whether to delete the repaired node from the queue.'},
+                  ),
+                  'returns': {'type': 'dict', 'desc': 'The queue node information'}}},
+    )
+    _storm_lib_path = ('model', 'migration', 's', 'model_0_2_31')
+
+    def getObjLocals(self):
+        return {
+            'listNodes': self._methListNodes,
+            'printNode': self._methPrintNode,
+            'repairNode': self._methRepairNode,
+        }
+
+    async def _hasCoreQueue(self, name):
+        try:
+            await self.runt.snap.core.getCoreQueue(name)
+            return True
+        except s_exc.NoSuchName:
+            return False
+
+    async def _methListNodes(self, form=None, source=None, offset=0, size=None):
+        form = await s_stormtypes.tostr(form, noneok=True)
+        source = await s_stormtypes.tostr(source, noneok=True)
+        offset = await s_stormtypes.toint(offset)
+        size = await s_stormtypes.toint(size, noneok=True)
+
+        if not await self._hasCoreQueue('model_0_2_31:nodes'):
+            await self.runt.printf('Queue model_0_2_31:nodes not found, no nodes to list.')
+            return
+
+        nodes = self.runt.snap.core.coreQueueGets('model_0_2_31:nodes', offs=offset, cull=False, size=size)
+        async for offs, node in nodes:
+            if form is not None and node['formname'] != form:
+                continue
+
+            if source is not None and source not in node['sources']:
+                continue
+
+            yield (offs, node['formname'], node['formvalu'], node['sources'])
+
+    async def _methPrintNode(self, offset):
+        offset = await s_stormtypes.toint(offset)
+
+        if not await self._hasCoreQueue('model_0_2_31:nodes'):
+            await self.runt.printf('Queue model_0_2_31:nodes not found, no nodes to print.')
+            return
+
+        node = await self.runt.snap.core.coreQueueGet('model_0_2_31:nodes', offs=offset, cull=False)
+        if not node:
+            await self.runt.warn(f'Queued node with offset {offset} not found.')
+            return
+
+        node = node[1]
+
+        await self.runt.printf(f'{node["formname"]}={repr(node["formvalu"])}')
+
+        for layriden, sode in node['sodes'].items():
+            await self.runt.printf(f'  layer: {layriden}')
+
+            for propname, propvalu in sode.get('props', {}).items():
+                if propname == '.seen':
+                    mintime, maxtime = propvalu[0]
+                    mindt = s_time.repr(mintime)
+                    maxdt = s_time.repr(maxtime)
+                    await self.runt.printf(f'    .seen = ({mindt}, {maxdt})')
+                else:
+                    await self.runt.printf(f'    :{propname} = {propvalu[0]}')
+
+            for tagname, tagvalu in sode.get('tags', {}).items():
+                if tagvalu == (None, None):
+                    await self.runt.printf(f'    #{tagname}')
+                else:
+                    mintime, maxtime = tagvalu
+                    mindt = s_time.repr(mintime)
+                    maxdt = s_time.repr(maxtime)
+                    await self.runt.printf(f'    #{tagname} = ({mindt}, {maxdt})')
+
+            for tagprop, tagpropvalu in sode.get('tagprops', {}).items():
+                for prop, valu in tagpropvalu.items():
+                    await self.runt.printf(f'    #{tagprop}:{prop} = {valu[0]}')
+
+        if sources := node['sources']:
+            await self.runt.printf(f'  sources: {sorted(sources)}')
+
+        if noderefs := node['refs']:
+            await self.runt.printf('  refs:')
+
+            for layriden, reflist in noderefs.items():
+                await self.runt.printf(f'    layer: {layriden}')
+                for iden, refinfo in reflist:
+                    form, prop, *_ = refinfo
+                    await self.runt.printf(f'      - {form}:{prop} (iden: {iden}')
+
+        n1edges = node['n1edges']
+        n2edges = node['n2edges']
+
+        if n1edges or n2edges:
+            await self.runt.printf('  edges:')
+
+        for layriden, edges in n1edges.items():
+            for verb, iden in edges:
+                await self.runt.printf(f'    -({verb})> {iden}')
+
+        for layriden, edges in n2edges.items():
+            for verb, iden, n2form in edges:
+                await self.runt.printf(f'    <({verb})- {iden}')
+
+    async def _repairNode(self, offset, newvalu):
+        item = await self.runt.snap.core.coreQueueGet('model_0_2_31:nodes', offset, cull=False)
+        if item is None:
+            await self.runt.warn(f'Queued node with offset {offset} not found.')
+            return False
+
+        node = item[1]
+
+        nodeform = node['formname']
+        form = self.runt.snap.core.model.form(nodeform)
+
+        norm, info = form.type.norm(newvalu)
+
+        buid = s_common.buid((nodeform, norm))
+
+        nodeedits = {}
+
+        for layriden in node['layers']:
+            nodeedits.setdefault(layriden, {})
+
+            layer = self.runt.snap.core.getLayer(layriden)
+            if layer is None: # pragma: no cover
+                await self.runt.warn(f'Layer does not exist to recreate node: {layriden}.')
+                return False
+
+        await self.runt.printf(f'Repairing node at offset {offset} from {node["formvalu"]} -> {norm}')
+
+        # Create the node in the right layers
+        for layriden in node['layers']:
+            nodeedits[layriden][buid] = (
+                buid, nodeform, [
+                (s_layer.EDIT_NODE_ADD, (norm, form.type.stortype), ()),
+            ])
+
+            for propname, propvalu in info.get('subs', {}).items():
+                prop = form.prop(propname)
+                if prop is None:
+                    continue
+
+                stortype = prop.type.stortype
+
+                nodeedits[layriden][buid][2].append(
+                    (s_layer.EDIT_PROP_SET, (propname, propvalu, None, stortype), ()),
+                )
+
+        for layriden, sode in node['sodes'].items():
+            nodeedits.setdefault(layriden, {})
+            nodeedits[layriden].setdefault(buid, (buid, nodeform, []))
+
+            for propname, propvalu in sode.get('props', {}).items():
+                propvalu, stortype = propvalu
+
+                nodeedits[layriden][buid][2].append(
+                    (s_layer.EDIT_PROP_SET, (propname, propvalu, None, stortype), ()),
+                )
+
+            for tagname, tagvalu in sode.get('tags', {}).items():
+                nodeedits[layriden][buid][2].append(
+                    (s_layer.EDIT_TAG_SET, (tagname, tagvalu, None), ()),
+                )
+
+            for tagprop, tagpropvalu in sode.get('tagprops', {}).items():
+                for propname, propvalu in tagpropvalu.items():
+                    propvalu, stortype = propvalu
+                    nodeedits[layriden][buid][2].append(
+                        (s_layer.EDIT_TAGPROP_SET, (tagname, propname, propvalu, None, stortype), ()),
+                    )
+
+        for layriden, data in node['nodedata'].items():
+            nodeedits.setdefault(layriden, {})
+            nodeedits[layriden].setdefault(buid, (buid, nodeform, []))
+
+            for name, valu in data:
+                nodeedits[layriden][buid][2].append(
+                    (s_layer.EDIT_NODEDATA_SET, (name, valu, None), ()),
+                )
+
+        for layriden, edges in node['n1edges'].items():
+            nodeedits.setdefault(layriden, {})
+            nodeedits[layriden].setdefault(buid, (buid, nodeform, []))
+
+            for verb, iden in edges:
+                nodeedits[layriden][buid][2].append(
+                    (s_layer.EDIT_EDGE_ADD, (verb, iden), ()),
+                )
+
+        for layriden, edges in node['n2edges'].items():
+            n1iden = s_common.ehex(buid)
+
+            for verb, iden, n2form in edges:
+                n2buid = s_common.uhex(iden)
+
+                nodeedits.setdefault(layriden, {})
+                nodeedits[layriden].setdefault(n2buid, (n2buid, n2form, []))
+
+                nodeedits[layriden][n2buid][2].append(
+                    (s_layer.EDIT_EDGE_ADD, (verb, n1iden), ()),
+                )
+
+        for layriden, reflist in node['refs'].items():
+            layer = self.runt.snap.core.getLayer(layriden)
+            if layer is None:
+                continue
+
+            for iden, refinfo in reflist:
+                refform, refprop, reftype, isarray, isro = refinfo
+
+                if isro:
+                    continue
+
+                refbuid = s_common.uhex(iden)
+
+                nodeedits.setdefault(layriden, {})
+                nodeedits[layriden].setdefault(refbuid, (refbuid, refform, []))
+
+                if reftype == 'ndef':
+                    propvalu = (nodeform, norm)
+                else:
+                    propvalu = norm
+
+                stortype = self.runt.snap.core.model.type(reftype).stortype
+
+                if isarray:
+
+                    sode = await layer.getStorNode(refbuid)
+                    if not sode:
+                        continue
+
+                    props = sode.get('props', {})
+
+                    curv, _ = props.get(refprop, (None, None))
+                    _curv = curv
+
+                    if _curv is None:
+                        _curv = []
+
+                    newv = list(_curv).copy()
+                    newv.append(propvalu)
+
+                    nodeedits[layriden][refbuid][2].append(
+                        (s_layer.EDIT_PROP_SET, (refprop, newv, curv, stortype | s_layer.STOR_FLAG_ARRAY), ()),
+                    )
+
+                else:
+
+                    nodeedits[layriden][refbuid][2].append(
+                        (s_layer.EDIT_PROP_SET, (refprop, propvalu, None, stortype), ()),
+                    )
+
+        meta = {'time': s_common.now(), 'user': self.runt.snap.core.auth.rootuser.iden}
+
+        # Process all layer edits as a single batch
+        for layriden, edits in nodeedits.items():
+            layer = self.runt.snap.core.getLayer(layriden)
+            if layer is None: # pragma: no cover
+                continue
+
+            await layer.storNodeEditsNoLift(list(edits.values()), meta)
+
+        return True
+
+    async def _methRepairNode(self, offset, newvalu, remove=False):
+        ok = False
+
+        if not await self._hasCoreQueue('model_0_2_31:nodes'):
+            await self.runt.printf('Queue model_0_2_31:nodes not found, no nodes to repair.')
+            return False
+
+        try:
+            ok = await self._repairNode(offset, newvalu)
+        except s_exc.SynErr as exc: # pragma: no cover
+            mesg = exc.get('mesg')
+            await self.runt.warn(f'Error when restoring node {offset}: {mesg}')
+
+        if ok and remove:
+            await self.runt.printf(f'Removing queued node: {offset}.')
+            await self.runt.snap.core.coreQueuePop('model_0_2_31:nodes', offset)
+
+        return ok

synapse/lib/stormlib/modelext.py

@@ -105,6 +105,21 @@ class LibModelExt(s_stormtypes.Lib):
                        'desc': 'The form of the n2 node. May be "*" or null to specify "any".'},
                   ),
                   'returns': {'type': 'null'}}},
+        {'name': 'addType', 'desc': 'Add an extended type definition to the data model.',
+         'type': {'type': 'function', '_funcname': 'addType',
+                  'args': (
+                      {'name': 'typename', 'type': 'str', 'desc': 'The name of the type to add.'},
+                      {'name': 'basetype', 'type': 'str', 'desc': 'The base type the type is derived from.'},
+                      {'name': 'typeopts', 'type': 'dict', 'desc': 'A Synapse type opts dictionary.'},
+                      {'name': 'typeinfo', 'type': 'dict', 'desc': 'A Synapse type info dictionary.'},
+                  ),
+                  'returns': {'type': 'null'}}},
+        {'name': 'delType', 'desc': 'Remove an extended type definition from the model.',
+         'type': {'type': 'function', '_funcname': 'delType',
+                  'args': (
+                      {'name': 'typename', 'type': 'str', 'desc': 'The extended type to remove.'},
+                  ),
+                  'returns': {'type': 'null'}}},
     )
     _storm_lib_path = ('model', 'ext')
 
@@ -122,6 +137,8 @@ class LibModelExt(s_stormtypes.Lib):
             'addExtModel': self.addExtModel,
             'addEdge': self.addEdge,
             'delEdge': self.delEdge,
+            'addType': self.addType,
+            'delType': self.delType,
         }
 
     # TODO type docs in the new convention
@@ -210,6 +227,7 @@ class LibModelExt(s_stormtypes.Lib):
         return await self.runt.snap.core.getExtModel()
 
     async def addExtModel(self, model):
+        self.runt.reqAdmin()
         model = await s_stormtypes.toprim(model)
         return await self.runt.snap.core.addExtModel(model)
 
@@ -249,3 +267,16 @@ class LibModelExt(s_stormtypes.Lib):
 
         s_stormtypes.confirm(('model', 'edge', 'del'))
         await self.runt.snap.core.delEdge((n1form, verb, n2form))
+
+    async def addType(self, typename, basetype, typeopts, typeinfo):
+        typename = await s_stormtypes.tostr(typename)
+        basetype = await s_stormtypes.tostr(basetype)
+        typeopts = await s_stormtypes.toprim(typeopts)
+        typeinfo = await s_stormtypes.toprim(typeinfo)
+        s_stormtypes.confirm(('model', 'type', 'add', typename))
+        await self.runt.snap.core.addType(typename, basetype, typeopts, typeinfo)
+
+    async def delType(self, typename):
+        typename = await s_stormtypes.tostr(typename)
+        s_stormtypes.confirm(('model', 'type', 'del', typename))
+        await self.runt.snap.core.delType(typename)

synapse/lib/stormlib/scrape.py

@@ -155,10 +155,7 @@ class LibScrape(s_stormtypes.Lib):
         if fangs:
             _fangs = {src: dst for (src, dst) in fangs}
             _fangre = s_scrape.genFangRegex(_fangs)
-            if len(text) < s_scrape.SCRAPE_SPAWN_LENGTH:
-                scrape_text, offsets = s_scrape.refang_text2(text, re=_fangre, fangs=_fangs)
-            else:
-                scrape_text, offsets = await s_coro.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
+            scrape_text, offsets = await s_coro.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
 
         async for info in s_scrape.genMatchesAsync(scrape_text, regx, opts=opts):
             valu = info.pop('valu')