synapse 2.186.0__py311-none-any.whl → 2.188.0__py311-none-any.whl

synapse/lib/nexus.py

@@ -341,6 +341,12 @@ class NexsRoot(s_base.Base):
         if meta is None:
             meta = {}
 
+        if (nexus := self._nexskids.get(nexsiden)) is None:
+            raise s_exc.NoSuchIden(mesg=f'No Nexus Pusher with iden {nexsiden}.', iden=nexsiden)
+
+        if event not in nexus._nexshands:
+            raise s_exc.NoSuchName(mesg=f'No Nexus handler for event {event}.', name=event)
+
         async with self.cell.nexslock:
             self.reqNotReadOnly()
             # Keep a reference to the shielded task to ensure it isn't GC'd

synapse/lib/node.py

@@ -173,11 +173,13 @@ class Node:
             if prop is None:
                 return None
 
-            if prop.modl.form(prop.type.name) is None:
+            if prop.modl.form(prop.type.name) is not None:
+                buid = s_common.buid((prop.type.name, valu))
+            elif prop.type.name == 'ndef':
+                buid = s_common.buid(valu)
+            else:
                 return None
 
-            buid = s_common.buid((prop.type.name, valu))
-
             step = cache.get(buid, s_common.novalu)
             if step is s_common.novalu:
                 step = cache[buid] = await node.snap.getNodeByBuid(buid)

synapse/lib/scrape.py

@@ -24,8 +24,6 @@ ipaddress = s_common.ipaddress
 
 logger = logging.getLogger(__name__)
 
-SCRAPE_SPAWN_LENGTH = 5000
-
 tldlist = list(s_data.get('iana.tlds'))
 tldlist.extend([
     'bit',
@@ -445,6 +443,9 @@ def _rewriteRawValu(text: str, offsets: dict, info: dict):
     info['match'] = match
     info['offset'] = baseoff + offset
 
+def _genMatchList(text: str, regx: regex.Regex, opts: dict):
+    return [info for info in _genMatches(text, regx, opts)]
+
 def _genMatches(text: str, regx: regex.Regex, opts: dict):
 
     cb = opts.get('callback')
@@ -504,23 +505,9 @@ def genMatches(text: str, regx: regex.Regex, opts: dict):
     for match in _genMatches(text, regx, opts):
         yield match
 
-def _spawn_genmatches(sock, text, regx, opts): # pragma: no cover
-    '''
-    Multiprocessing target for generating matches.
-    '''
-    try:
-        for info in _genMatches(text, regx, opts):
-            sock.sendall(s_msgpack.en((True, info)))
-
-        sock.sendall(s_msgpack.en((True, None)))
-
-    except Exception as e:
-        mesg = s_common.retnexc(e)
-        sock.sendall(s_msgpack.en(mesg))
-
 async def genMatchesAsync(text: str, regx: regex.Regex, opts: dict):
     '''
-    Generate regular expression matches for a blob of text, potentially in a spawned process.
+    Generate regular expression matches for a blob of text, using the shared forked process pool.
 
     Args:
         text (str): The text to generate matches for.
@@ -547,30 +534,9 @@ async def genMatchesAsync(text: str, regx: regex.Regex, opts: dict):
     Yields:
         dict: A dictionary of match results.
     '''
-    if len(text) < SCRAPE_SPAWN_LENGTH:
-        for match in _genMatches(text, regx, opts):
-            yield match
-        return
-
-    link00, sock00 = await s_link.linksock()
-
-    try:
-        async with link00:
-
-            todo = s_common.todo(_spawn_genmatches, sock00, text, regx, opts)
-            link00.schedCoro(s_coro.spawn(todo, log_conf=s_common._getLogConfFromEnv()))
-
-            while (mesg := await link00.rx()) is not None:
-
-                info = s_common.result(mesg)
-                if info is None:
-                    return
-
-                yield info
-
-    finally:
-        sock00.close()
-
+    matches = await s_coro.semafork(_genMatchList, text, regx, opts)
+    for info in matches:
+        yield info
 
 def _contextMatches(scrape_text, text, ruletype, refang, offsets):
 
@@ -585,33 +551,16 @@ def _contextMatches(scrape_text, text, ruletype, refang, offsets):
 
                 yield info
 
-def _contextScrape(text, form=None, refang=True, first=False):
-    scrape_text = text
-    offsets = {}
-    if refang:
-        scrape_text, offsets = refang_text2(text)
-
-    for ruletype, blobs in _regexes.items():
-        if form and form != ruletype:
-            continue
+def _contextScrapeList(text, form=None, refang=True, first=False):
+    return [info for info in _contextScrape(text, form=form, refang=refang, first=first)]
 
-        for info in _contextMatches(scrape_text, text, ruletype, refang, offsets):
-
-            yield info
-
-            if first:
-                return
-
-async def _contextScrapeAsync(text, form=None, refang=True, first=False):
+def _contextScrape(text, form=None, refang=True, first=False):
     scrape_text = text
     offsets = {}
     if refang:
         scrape_text, offsets = refang_text2(text)
 
     for ruletype, blobs in _regexes.items():
-
-        await asyncio.sleep(0)
-
         if form and form != ruletype:
             continue
 
@@ -666,27 +615,12 @@ def scrape(text, ptype=None, refang=True, first=False):
     Returns:
         (str, object): Yield tuples of node ndef values.
     '''
-
-    for info in contextScrape(text, form=ptype, refang=refang, first=first):
+    for info in _contextScrape(text, form=ptype, refang=refang, first=first):
         yield info.get('form'), info.get('valu')
 
-def _spawn_scrape(sock, text, form=None, refang=True, first=False): # pragma: no cover
-    '''
-    Multiprocessing target for scraping text.
-    '''
-    try:
-        for info in _contextScrape(text, form=form, refang=refang, first=first):
-            sock.sendall(s_msgpack.en((True, info)))
-
-        sock.sendall(s_msgpack.en((True, None)))
-
-    except Exception as e:
-        mesg = s_common.retnexc(e)
-        sock.sendall(s_msgpack.en(mesg))
-
 async def contextScrapeAsync(text, form=None, refang=True, first=False):
     '''
-    Scrape types from a blob of text and yield info dictionaries, potentially in a spawned process.
+    Scrape types from a blob of text and yield info dictionaries, using the shared forked process pool.
 
     Args:
         text (str): Text to scrape.
@@ -712,33 +646,13 @@ async def contextScrapeAsync(text, form=None, refang=True, first=False):
     Returns:
         (dict): Yield info dicts of results.
     '''
-    if len(text) < SCRAPE_SPAWN_LENGTH:
-        async for info in _contextScrapeAsync(text, form=form, refang=refang, first=first):
-            yield info
-        return
-
-    link00, sock00 = await s_link.linksock()
-
-    try:
-        async with link00:
-
-            todo = s_common.todo(_spawn_scrape, sock00, text, form=form, refang=refang, first=first)
-            link00.schedCoro(s_coro.spawn(todo, log_conf=s_common._getLogConfFromEnv()))
-
-            while (mesg := await link00.rx()) is not None:
-
-                info = s_common.result(mesg)
-                if info is None:
-                    return
-
-                yield info
-
-    finally:
-        sock00.close()
+    matches = await s_coro.semafork(_contextScrapeList, text, form=form, refang=refang, first=first)
+    for info in matches:
+        yield info
 
 async def scrapeAsync(text, ptype=None, refang=True, first=False):
     '''
-    Scrape types from a blob of text and return node tuples, potentially in a spawned process.
+    Scrape types from a blob of text and return node tuples, using the shared forked process pool.
 
     Args:
         text (str): Text to scrape.
@@ -749,6 +663,6 @@ async def scrapeAsync(text, ptype=None, refang=True, first=False):
     Returns:
         (str, object): Yield tuples of node ndef values.
     '''
-
-    async for info in contextScrapeAsync(text, form=ptype, refang=refang, first=first):
+    matches = await s_coro.semafork(_contextScrapeList, text, form=ptype, refang=refang, first=first)
+    for info in matches:
         yield info.get('form'), info.get('valu')

synapse/lib/spooled.py

@@ -7,6 +7,8 @@ import synapse.lib.const as s_const
 import synapse.lib.msgpack as s_msgpack
 import synapse.lib.lmdbslab as s_lmdbslab
 
+MAX_SPOOL_SIZE = 10000
+
 class Spooled(s_base.Base):
     '''
     A Base class that can be used to implement objects which fallback to lmdb.
@@ -15,7 +17,7 @@ class Spooled(s_base.Base):
     together. Under memory pressure, these objects have a better shot of getting paged out.
     '''
 
-    async def __anit__(self, dirn=None, size=10000, cell=None):
+    async def __anit__(self, dirn=None, size=MAX_SPOOL_SIZE, cell=None):
         '''
         Args:
             dirn(Optional[str]): base directory used for backing slab.  If None, system temporary directory is used
@@ -55,7 +57,7 @@ class Set(Spooled):
     A minimal set-like implementation that will spool to a slab on large growth.
     '''
 
-    async def __anit__(self, dirn=None, size=10000, cell=None):
+    async def __anit__(self, dirn=None, size=MAX_SPOOL_SIZE, cell=None):
         await Spooled.__anit__(self, dirn=dirn, size=size, cell=cell)
         self.realset = set()
         self.len = 0
@@ -84,6 +86,27 @@ class Set(Spooled):
 
         return len(self.realset)
 
+    async def copy(self):
+        newset = await Set.anit(dirn=self.dirn, size=self.size, cell=self.cell)
+
+        if self.fallback:
+            await newset._initFallBack()
+            await self.slab.copydb(None, newset.slab)
+            newset.len = self.len
+
+        else:
+            newset.realset = self.realset.copy()
+
+        return newset
+
+    async def clear(self):
+        if self.fallback:
+            self.len = 0
+            await self.slab.trash()
+            await self._initFallBack()
+        else:
+            self.realset.clear()
+
     async def add(self, valu):
 
         if self.fallback:
@@ -117,7 +140,7 @@ class Set(Spooled):
 
 class Dict(Spooled):
 
-    async def __anit__(self, dirn=None, size=10000, cell=None):
+    async def __anit__(self, dirn=None, size=MAX_SPOOL_SIZE, cell=None):
 
         await Spooled.__anit__(self, dirn=dirn, size=size, cell=cell)
         self.realdict = {}

synapse/lib/storm.py

@@ -1357,30 +1357,42 @@ stormcmds = (
         'descr': "List existing cron jobs in the cortex.",
         'cmdargs': (),
         'storm': '''
+            init {
+                $conf = ({
+                    "columns": [
+                        {"name": "user", "width": 24},
+                        {"name": "iden", "width": 10},
+                        {"name": "view", "width": 10},
+                        {"name": "en?", "width": 3},
+                        {"name": "rpt?", "width": 4},
+                        {"name": "now?", "width": 4},
+                        {"name": "err?", "width": 4},
+                        {"name": "# start", "width": 7},
+                        {"name": "last start", "width": 16},
+                        {"name": "last end", "width": 16},
+                        {"name": "query", "newlines": "split"},
+                    ],
+                    "separators": {
+                        "row:outline": false,
+                        "column:outline": false,
+                        "header:row": "#",
+                        "data:row": "",
+                        "column": "",
+                        },
+                })
+                $printer = $lib.tabular.printer($conf)
+            }
             $crons = $lib.cron.list()
-
             if $crons {
-                $lib.print("user       iden       view       en? rpt? now? err? # start last start       last end         query")
-
+                $lib.print($printer.header())
                 for $cron in $crons {
-
                     $job = $cron.pprint()
-
-                    $user = $job.user.ljust(10)
-                    $view = $job.viewshort.ljust(10)
-                    $iden = $job.idenshort.ljust(10)
-                    $enabled = $job.enabled.ljust(3)
-                    $isrecur = $job.isrecur.ljust(4)
-                    $isrunning = $job.isrunning.ljust(4)
-                    $iserr = $job.iserr.ljust(4)
-                    $startcount = $lib.str.format("{startcount}", startcount=$job.startcount).ljust(7)
-                    $laststart = $job.laststart.ljust(16)
-                    $lastend = $job.lastend.ljust(16)
-
-       $lib.print("{user} {iden} {view} {enabled} {isrecur} {isrunning} {iserr} {startcount} {laststart} {lastend} {query}",
-                               user=$user, iden=$iden, view=$view, enabled=$enabled, isrecur=$isrecur,
-                               isrunning=$isrunning, iserr=$iserr, startcount=$startcount,
-                               laststart=$laststart, lastend=$lastend, query=$job.query)
+                    $row = (
+                        $job.user, $job.idenshort, $job.viewshort, $job.enabled,
+                        $job.isrecur, $job.isrunning, $job.iserr, `{$job.startcount}`,
+                        $job.laststart, $job.lastend, $job.query
+                    )
+                    $lib.print($printer.row($row))
                 }
             } else {
                 $lib.print("No cron jobs found")
@@ -5268,6 +5280,13 @@ class BackgroundCmd(Cmd):
         async for item in genr:
             yield item
 
+        _query = await s_stormtypes.tostr(self.opts.query)
+        query = await runt.getStormQuery(_query)
+
+        # make sure the subquery *could* have run
+        async with runt.getSubRuntime(query) as subr:
+            query.validate(subr)
+
         runtprims = await s_stormtypes.toprim(self.runt.getScopeVars(), use_list=True)
         runtvars = {k: v for (k, v) in runtprims.items() if s_msgpack.isok(v)}
 
@@ -5277,12 +5296,6 @@ class BackgroundCmd(Cmd):
             'vars': runtvars,
         }
 
-        _query = await s_stormtypes.tostr(self.opts.query)
-        query = await runt.getStormQuery(_query)
-
-        # make sure the subquery *could* have run with existing vars
-        query.validate(runt)
-
         coro = self.execStormTask(query, opts)
         runt.snap.core.schedCoro(coro)
 
@@ -5339,9 +5352,12 @@ class ParallelCmd(Cmd):
             raise s_exc.StormRuntimeError(mesg=mesg)
 
         size = await s_stormtypes.toint(self.opts.size)
-        query = await runt.getStormQuery(self.opts.query)
 
-        query.validate(runt)
+        _query = await s_stormtypes.tostr(self.opts.query)
+        query = await runt.getStormQuery(_query)
+
+        async with runt.getSubRuntime(query) as subr:
+            query.validate(subr)
 
         async with await s_base.Base.anit() as base:
 
@@ -5979,6 +5995,13 @@ class RunAsCmd(Cmd):
 
     NOTE: This command requires admin privileges.
 
+    NOTE: Heavy objects (for example a View or Layer) are bound to the context which they
+          are instantiated in and methods on them will be run using the user in that
+          context. This means that executing a method on a variable containing a heavy
+          object which was instantiated outside of the runas command and then used
+          within the runas command will check the permissions of the outer user, not
+          the one specified by the runas command.
+
     Examples:
 
         // Create a node as another user.