synapse 2.180.1__py311-none-any.whl → 2.182.0__py311-none-any.whl

synapse/assets/__init__.py

@@ -0,0 +1,35 @@
+import os
+import logging
+
+import synapse.common as s_common
+
+logger = logging.getLogger(__name__)
+dirname = os.path.dirname(__file__)
+
+def getStorm(*names):
+    '''
+    Return a storm file from the synapse storm folder.
+
+    Example:
+
+        text = storm.get('migrate.storm')
+        await core.callStorm(text)
+
+    Example #2:
+        text = storm.get('migrations', 'model-0.2.28.storm')
+        await core.callStorm(text)
+    '''
+    fp = getAssetPath('storm', *names)
+    with open(fp, 'rb') as fd:
+        text = fd.read()
+    return text.decode('utf8')
+
+def getAssetPath(*names):
+    fp = s_common.genpath(dirname, *names)
+    if not fp.startswith(dirname):
+        logger.error(f'{fp} is not in {dirname}')
+        raise ValueError(f'Path escaping detected for {names}')
+    if not os.path.isfile(fp):
+        logger.error('{} does not exist'.format(fp))
+        raise ValueError(f'Asset does not exist for {names}')
+    return fp

synapse/assets/storm/migrations/model-0.2.28.storm

@@ -0,0 +1,355 @@
+$layers = $lib.spooled.set()
+$layers.adds($layridens)
+
+$refinfoCache = ({})
+
+function getRefInfo(formname) {
+    $cached = $refinfoCache.$formname
+    if $cached { return($cached) }
+
+    $refinfo = ([])
+
+    syn:prop:type*in=($formname, array, ndef)
+    // Don't include the form we're querying
+    -:form = $formname
+
+    $prop = $lib.model.prop($node.repr())
+    $proptype = $prop.type.name
+
+    // Don't include array types that don't have a base type of $formname or ndef
+    -$($proptype = 'array' and not ([$formname, 'ndef']).has($prop.type.opts.type))
+
+    $isarray = $lib.false
+
+    if ($proptype = 'array') {
+        $isarray = $lib.true
+        $proptype = $prop.type.opts.type
+    }
+
+    $refinfo.append((:form, $prop.name, $proptype, $isarray))
+
+    fini {
+        $refinfoCache.$formname = $refinfo
+        return($refinfo)
+    }
+}
+
+function removeNode(n, viewiden, layriden) {
+
+    $nodeq = $lib.queue.gen("model_0_2_28:nodes")
+    $refsq = $lib.queue.gen("model_0_2_28:nodes:refs")
+    $edgeq = $lib.queue.gen("model_0_2_28:nodes:edges")
+
+    $sources = ([])
+
+    $iden = $n.iden()
+    $repr = $n.repr()
+    $form = $n.form()
+    $ndef = $n.ndef()
+
+    $lib.log.debug(`Saving invalid {$form} node info: {$repr}`)
+
+    $refinfos = $getRefInfo($form)
+
+    for $view in $lib.view.list(deporder=$lib.true) {
+        if (not $layers.has($view.layers.0.iden)) { continue }
+
+        view.exec $view.iden {
+            $edges = ([])
+            $references = ([])
+
+            // Get references and store them in queue
+            for $refinfo in $refinfos {
+
+                ($formname, $propname, $proptype, $isarray) = $refinfo
+                if ($proptype = "ndef") {
+                    $oldvalu = $ndef
+                } else {
+                    $oldvalu = $repr
+                }
+
+                for $ref in $lib.model.migration.liftByPropValuNoNorm($formname, $propname, $oldvalu) {
+                    yield $ref
+
+                    if $isarray {
+
+                        // We can't just [ :$propname-=$oldvalu ] because the norm() function gets called on the array type
+                        // deep down in the AST. So, instead, we have to operate on the whole array.
+
+                        $list = $lib.copy(:$propname)
+                        $list.rem($oldvalu, all=$lib.true)
+
+                        if $list {
+                            $lib.model.migration.setNodePropValuNoNorm($node, $propname, $list)
+                        } else {
+                            [ -:$propname ]
+                        }
+
+                    } else {
+                        try {
+                            [ -:$propname ]
+                        } catch ReadOnlyProp as exc {
+                            // The property is readonly so we can only delete it
+                            $lib.log.warning(`{$formname}:{$propname} is readonly, cannot modify. Storing node: {$node.repr()}`)
+                            $removeNode($node, $view.iden, $view.layers.0.iden)
+                            | delnode |
+                            continue
+                        }
+                    }
+
+                    $ref = ({
+                        "iden": $node.iden(),
+                        "refinfo": $refinfo,
+                    })
+
+                    $references.append($ref)
+
+                    // Flush to the queue if the list grows too large
+                    if ($references.size() > 1000) { // pragma: no cover start
+                        $item = ({
+                            "iden": $iden,
+                            "view": $view.iden,
+                            "refs": $references,
+                        })
+
+                        $refsq.put($item)
+                        $references = ([])
+                    } // pragma: no cover stop
+                }
+            }
+
+            if $references {
+                $item = ({
+                    "iden": $iden,
+                    "view": $view.iden,
+                    "refs": $references,
+                })
+
+                $refsq.put($item)
+                $references = ([])
+            }
+
+            spin |
+
+            yield $iden |
+
+            // Get sources and store them in the queue
+            // Do this before we store/delete edges.
+            { <(seen)- meta:source
+                $sources.append($node.repr())
+            }
+
+            // Get N1 edges and store them in the queue
+            { for $edge in $node.edges() {
+                ($verb, $dst) = $edge
+                $edges.append($edge)
+
+                [ -($verb)> { yield $dst } ]
+
+                // Flush to the queue if the list grows too large
+                if ($edges.size() > 1000) { // pragma: no cover start
+                    $item = ({
+                        "iden": $iden,
+                        "view": $view.iden,
+                        "direction": "n1",
+                        "edges": $edges,
+                    })
+
+                    $edgeq.put($item)
+                    $edges = ([])
+                } // pragma: no cover stop
+            }}
+
+            if $edges {
+                $item = ({
+                    "iden": $iden,
+                    "view": $view.iden,
+                    "direction": "n1",
+                    "edges": $edges,
+                })
+
+                $edgeq.put($item)
+                $edges = ([])
+            }
+
+            // Get N2 edges and store them in the queue
+            { for $edge in $node.edges(reverse=$lib.true) {
+                ($verb, $src) = $edge
+                $edges.append($edge)
+
+                [ <($verb)- { yield $src } ]
+
+                // Flush to the queue if the list grows too large
+                if ($edges.size() > 1000) { // pragma: no cover start
+                    $item = ({
+                        "iden": $iden,
+                        "view": $view.iden,
+                        "direction": "n2",
+                        "edges": $edges,
+                    })
+
+                    $edgeq.put($item)
+                    $edges = ([])
+                } // pragma: no cover stop
+            }}
+
+            if $edges {
+                $item = ({
+                    "iden": $iden,
+                    "view": $view.iden,
+                    "direction": "n2",
+                    "edges": $edges,
+                })
+
+                $edgeq.put($item)
+                $edges = ([])
+            }
+        }
+    }
+
+    $item = ({
+        "iden": $iden,
+        "form": $form,
+        "repr": $repr,
+        "props": $n.props,
+        "view": $viewiden,
+        "layer": $layriden,
+        "tags": $n.tags(),
+        "data": $n.data.list(),
+        "sources": $sources,
+    })
+
+    $nodeq.put($item)
+
+    return()
+}
+
+function migrateCpeNode(oldcpe, newcpe) {
+
+    // migrate the edges, tags, node data, and extended props
+    $lib.model.migration.copyEdges($oldcpe, $newcpe)
+    $lib.model.migration.copyTags($oldcpe, $newcpe)
+    $lib.model.migration.copyData($oldcpe, $newcpe)
+    $lib.model.migration.copyExtProps($oldcpe, $newcpe)
+
+    $refinfos = $getRefInfo($oldcpe.form())
+
+    // Iterate through the views and fix up all the references
+    for $view in $lib.view.list(deporder=$lib.true) {
+        if (not $layers.has($view.layers.0.iden)) { continue }
+
+        view.exec $view.iden {
+
+            // Fix references that point to old node to now point to new node
+            for $refinfo in $refinfos {
+                ($form, $prop, $proptype, $isarray) = $refinfo
+                if ($proptype = "ndef") {
+                    $oldvalu = $oldcpe.ndef()
+                    $newvalu = $newcpe.ndef()
+                } else {
+                    $oldvalu = $oldcpe.repr()
+                    $newvalu = $newcpe.repr()
+                }
+
+                for $n in $lib.model.migration.liftByPropValuNoNorm($form, $prop, $oldvalu) {
+                    yield $n
+
+                    if $isarray {
+
+                        // We can't just [ :$prop-=$oldvalu :$prop+=$newvalu ] because the norm() function gets called
+                        // on the array type deep down in the AST. So, instead, we have to operate on the whole array.
+
+                        $list = $lib.copy(:$prop)
+                        $list.rem($oldvalu, all=$lib.true)
+
+                        if (not $list.has($newvalu)) {
+                            $list.append($newvalu)
+                        }
+
+                        $lib.model.migration.setNodePropValuNoNorm($node, $prop, $list)
+
+                    } else {
+                        try {
+                            [ -:$prop :$prop=$newvalu ]
+                        } catch ReadOnlyProp as exc {
+                            // The property is readonly so we can only delete it
+                            $lib.log.warning(`{$form}:{$prop} is readonly, cannot modify. Storing node: {$node.repr()}`)
+                            $removeNode($node, $view.iden, $view.layers.0.iden)
+                            | delnode
+
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    return()
+}
+
+// This is a separate function because I was getting weird bugs from scoping
+// issues. Mainly, the $view variable being overwritten because this code was
+// scoped to the whole script.
+function main() {
+
+    for $view in $lib.view.list(deporder=$lib.true) {
+
+        if (not $layers.has($view.layers.0.iden)) { continue }
+
+        view.exec $view.iden {
+
+            $layer = $lib.layer.get()
+
+            for $oldcpe in $layer.liftByProp(it:sec:cpe) {
+                $info = $lib.model.migration.s.itSecCpe_2_170_0_internal($oldcpe)
+
+                if ($info.status = "success") {
+                    // No primary property changes, nothing to do. Node has been fully migrated.
+                    if (not $info.valu) {
+                        continue
+                    }
+
+                    /*
+                     * At this point, we have a node that can be fixed but
+                     * needs to be migrated to a new node because the primary
+                     * property needs to be changed. We'll create a new
+                     * (correct) node, and copy everything from the old node.
+                     * Then we complete the migration by iterating through
+                     * all the views to fix the references.
+                     */
+
+                    $lib.log.debug(`Migrating invalid it:sec:cpe node: {$oldcpe.repr()} -> {$info.valu}`)
+
+                    [ it:sec:cpe=$info.valu
+                        .seen ?= $oldcpe.props.".seen"
+                    ]
+
+                    $migrateCpeNode($oldcpe, $node)
+
+                    spin |
+
+                    yield $oldcpe |
+                    delnode --deledges --force
+
+                } else {
+
+                    // Node could not be automatically migrated. Collect
+                    // critical information to eventually reconstruct this node
+                    // and store it in a queue.
+
+                    $removeNode($oldcpe, $view.iden, $view.layers.0.iden)
+
+                    yield $oldcpe |
+                    delnode --deledges --force
+                }
+            }
+        }
+    }
+
+    return()
+}
+
+// Pre-cache the it:sec:cpe references
+$getRefInfo(it:sec:cpe)
+
+$main()

synapse/common.py

@@ -2,6 +2,7 @@ import io
 import os
 import ssl
 import sys
+import enum
 import json
 import http
 import stat
@@ -36,7 +37,7 @@ import synapse.lib.const as s_const
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.structlog as s_structlog
 
-import enum
+import synapse.vendor.cpython.lib.ipaddress as ipaddress
 
 try:
     from yaml import CSafeLoader as Loader

synapse/cortex.py

@@ -115,6 +115,8 @@ SYNC_NODEEDIT = 1   # A nodeedit:  (<offs>, 0, <etyp>, (<etype args>))
 SYNC_LAYR_ADD = 3   # A layer was added
 SYNC_LAYR_DEL = 4   # A layer was deleted
 
+MAX_NEXUS_DELTA = 3_600
+
 reqValidTagModel = s_config.getJsValidator({
     'type': 'object',
     'properties': {
@@ -2278,31 +2280,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             if sodelist is not None:
                 yield sodelist
 
-    async def _mergeSodesUniq(self, layers, genrs, cmprkey, filtercmpr=None, reverse=False):
-        lastbuid = None
-        sodes = {}
-        async with await s_spooled.Set.anit(dirn=self.dirn) as uniqset:
-            async for layr, (_, buid), sode in s_common.merggenr2(genrs, cmprkey, reverse=reverse):
-                if buid in uniqset:
-                    continue
-
-                if not buid == lastbuid or layr in sodes:
-                    if lastbuid is not None:
-                        sodelist = await self._genSodeList(lastbuid, sodes, layers, filtercmpr)
-                        if sodelist is not None:
-                            yield sodelist
-                        sodes.clear()
-
-                    await uniqset.add(lastbuid)
-                    lastbuid = buid
-
-                sodes[layr] = sode
-
-            if lastbuid is not None:
-                sodelist = await self._genSodeList(lastbuid, sodes, layers, filtercmpr)
-                if sodelist is not None:
-                    yield sodelist
-
     async def _liftByDataName(self, name, layers):
         if len(layers) == 1:
             layr = layers[0].iden
@@ -2328,7 +2305,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for layr in layers:
             genrs.append(wrap_liftgenr(layr.iden, layr.liftByProp(form, prop, reverse=reverse)))
 
-        async for sodes in self._mergeSodesUniq(layers, genrs, cmprkey_indx, reverse=reverse):
+        def filtercmpr(sode):
+            if (props := sode.get('props')) is None:
+                return False
+
+            return props.get(prop) is not None
+
+        async for sodes in self._mergeSodes(layers, genrs, cmprkey_indx, filtercmpr, reverse=reverse):
             yield sodes
 
     async def _liftByPropValu(self, form, prop, cmprvals, layers, reverse=False):
@@ -2445,7 +2428,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         for layr in layers:
             genrs.append(wrap_liftgenr(layr.iden, layr.liftByTagProp(form, tag, prop, reverse=reverse)))
 
-        async for sodes in self._mergeSodesUniq(layers, genrs, cmprkey_indx, reverse=reverse):
+        def filtercmpr(sode):
+            if (tagprops := sode.get('tagprops')) is None:
+                return False
+
+            if (props := tagprops.get(tag)) is None:
+                return False
+
+            return props.get(prop) is not None
+
+        async for sodes in self._mergeSodes(layers, genrs, cmprkey_indx, filtercmpr, reverse=reverse):
             yield sodes
 
     async def _liftByTagPropValu(self, form, tag, prop, cmprvals, layers, reverse=False):
@@ -4401,7 +4393,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         except Exception as e:
             mesg = f'Invalid path for Extended HTTP API - cannot compile regular expression for [{path}] : {e}'
             raise s_exc.BadArg(mesg=mesg) from None
-        adef['iden'] = s_common.guid()
+
+        if adef.get('iden') is None:
+            adef['iden'] = s_common.guid()
+
+        iden = adef['iden']
+        if self._exthttpapis.get(iden) is not None:
+            raise s_exc.DupIden(mesg=f'Duplicate iden specified for Extended HTTP API: {iden}', iden=iden)
+
         adef['created'] = s_common.now()
         adef['updated'] = adef['created']
         adef = s_schemas.reqValidHttpExtAPIConf(adef)
@@ -5714,7 +5713,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if self.stormpool is not None and opts.get('mirror', True):
             extra = await self.getLogExtra(text=text)
-            proxy = await self._getMirrorProxy()
+            proxy = await self._getMirrorProxy(opts)
 
             if proxy is not None:
                 logger.info(f'Offloading Storm query {{{text}}} to mirror.', extra=extra)
@@ -5741,13 +5740,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         return i
 
     async def _getMirrorOpts(self, opts):
+        assert 'nexsoffs' in opts
         mirropts = s_msgpack.deepcopy(opts)
         mirropts['mirror'] = False
-        mirropts['nexsoffs'] = (await self.getNexsIndx() - 1)
         mirropts['nexstimeout'] = self.stormpoolopts.get('timeout:sync')
         return mirropts
 
-    async def _getMirrorProxy(self):
+    async def _getMirrorProxy(self, opts):
 
         if self.stormpool is None:  # pragma: no cover
             return None
@@ -5756,6 +5755,8 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
             logger.warning('Storm query mirror pool is empty, running query locally.')
             return None
 
+        proxy = None
+
         try:
             timeout = self.stormpoolopts.get('timeout:connection')
             proxy = await self.stormpool.proxy(timeout=timeout)
@@ -5764,10 +5765,23 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
                 # we are part of the pool and were selected. Convert to local use.
                 return None
 
+            curoffs = opts.setdefault('nexsoffs', await self.getNexsIndx() - 1)
+            miroffs = await s_common.wait_for(proxy.getNexsIndx(), timeout) - 1
+            if (delta := curoffs - miroffs) > MAX_NEXUS_DELTA:
+                mesg = (f'Pool mirror [{proxyname}] Nexus offset delta too large '
+                        f'({delta} > {MAX_NEXUS_DELTA}), running query locally.')
+                logger.warning(mesg, extra=await self.getLogExtra(delta=delta, mirror=proxyname, mirror_offset=miroffs))
+                return None
+
             return proxy
 
         except (TimeoutError, s_exc.IsFini):
-            logger.warning('Timeout waiting for pool mirror, running query locally.')
+            if proxy is None:
+                logger.warning('Timeout waiting for pool mirror, running query locally.')
+            else:
+                mesg = f'Timeout waiting for pool mirror [{proxyname}] Nexus offset, running query locally.'
+                logger.warning(mesg, extra=await self.getLogExtra(mirror=proxyname))
+                await proxy.fini()
             return None
 
     async def storm(self, text, opts=None):
@@ -5776,7 +5790,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if self.stormpool is not None and opts.get('mirror', True):
             extra = await self.getLogExtra(text=text)
-            proxy = await self._getMirrorProxy()
+            proxy = await self._getMirrorProxy(opts)
 
             if proxy is not None:
                 logger.info(f'Offloading Storm query {{{text}}} to mirror.', extra=extra)
@@ -5806,7 +5820,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if self.stormpool is not None and opts.get('mirror', True):
             extra = await self.getLogExtra(text=text)
-            proxy = await self._getMirrorProxy()
+            proxy = await self._getMirrorProxy(opts)
 
             if proxy is not None:
                 logger.info(f'Offloading Storm query {{{text}}} to mirror.', extra=extra)
@@ -5831,7 +5845,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
 
         if self.stormpool is not None and opts.get('mirror', True):
             extra = await self.getLogExtra(text=text)
-            proxy = await self._getMirrorProxy()
+            proxy = await self._getMirrorProxy(opts)
 
             if proxy is not None:
                 logger.info(f'Offloading Storm query {{{text}}} to mirror.', extra=extra)

synapse/cryotank.py

@@ -81,7 +81,7 @@ class CryoTank(s_base.Base):
         size = 0
 
         for chunk in s_common.chunks(items, 1000):
-            metrics = self._items.save(chunk)
+            metrics = await self._items.save(chunk)
             self._metrics.add(metrics)
             await self.fire('cryotank:puts', numrecords=len(chunk))
             size += len(chunk)

synapse/datamodel.py

@@ -124,6 +124,7 @@ class Prop:
         self.type = None
         self.typedef = typedef
 
+        self.alts = None
         self.locked = False
         self.deprecated = self.info.get('deprecated', False)
 
@@ -250,6 +251,18 @@ class Prop:
 
         return (buid, {'props': pnorms, 'ndef': ndef})
 
+    def getAlts(self):
+        '''
+        Return a list of Prop instances that are considered
+        alternative locations for our property value, including
+        self.
+        '''
+        if self.alts is None:
+            self.alts = [self]
+            for name in self.info.get('alts', ()):
+                self.alts.append(self.form.reqProp(name))
+        return self.alts
+
 class Form:
     '''
     The Form class implements data model logic for a node form.
@@ -432,6 +445,15 @@ class Form:
         '''
         return self.props.get(name)
 
+    def reqProp(self, name):
+        prop = self.props.get(name)
+        if prop is not None:
+            return prop
+
+        full = f'{self.name}:{name}'
+        mesg = f'No property named {full}.'
+        raise s_exc.NoSuchProp(mesg=mesg, name=full)
+
     def pack(self):
         props = {p.name: p.pack() for p in self.props.values()}
         info = {
@@ -1160,6 +1182,14 @@ class Model:
     def form(self, name):
         return self.forms.get(name)
 
+    def reqForm(self, name):
+        form = self.forms.get(name)
+        if form is not None:
+            return form
+
+        mesg = f'No form named {name}.'
+        raise s_exc.NoSuchForm(mesg=mesg, name=name)
+
     def univ(self, name):
         return self.univs.get(name)