synapse 2.232.0__tar.gz → 2.233.0__tar.gz

{synapse-2.232.0/synapse.egg-info → synapse-2.233.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse
-Version: 2.232.0
+Version: 2.233.0
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License-Expression: Apache-2.0

{synapse-2.232.0 → synapse-2.233.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.232.0'
+version = '2.233.0'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]

{synapse-2.232.0 → synapse-2.233.0}/synapse/axon.py

@@ -13,7 +13,6 @@ import synapse.exc as s_exc
 import synapse.common as s_common
 
 import synapse.lib.cell as s_cell
-import synapse.lib.coro as s_coro
 import synapse.lib.base as s_base
 import synapse.lib.json as s_json
 import synapse.lib.link as s_link
@@ -25,6 +24,7 @@ import synapse.lib.hashset as s_hashset
 import synapse.lib.httpapi as s_httpapi
 import synapse.lib.urlhelp as s_urlhelp
 import synapse.lib.msgpack as s_msgpack
+import synapse.lib.process as s_process
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.slabseqn as s_slabseqn
 
@@ -1447,7 +1447,7 @@ class Axon(s_cell.Cell):
             todo = s_common.todo(_spawn_readlines, sock00, errors=errors)
             async with await s_base.Base.anit() as scope:
 
-                scope.schedCoro(s_coro.spawn(todo, log_conf=await self._getSpawnLogConf()))
+                scope.schedCoro(s_process.spawn(todo, log_conf=await self._getSpawnLogConf()))
                 feedtask = scope.schedCoro(self._sha256ToLink(sha256, link00))
 
                 while not self.isfini:
@@ -1481,7 +1481,7 @@ class Axon(s_cell.Cell):
             todo = s_common.todo(_spawn_readrows, sock00, dialect, fmtparams, errors=errors)
             async with await s_base.Base.anit() as scope:
 
-                scope.schedCoro(s_coro.spawn(todo, log_conf=await self._getSpawnLogConf()))
+                scope.schedCoro(s_process.spawn(todo, log_conf=await self._getSpawnLogConf()))
                 feedtask = scope.schedCoro(self._sha256ToLink(sha256, link00))
 
                 while not self.isfini:

{synapse-2.232.0 → synapse-2.233.0}/synapse/common.py

@@ -23,7 +23,6 @@ import tarfile
 import binascii
 import builtins
 import tempfile
-import warnings
 import functools
 import itertools
 import threading

{synapse-2.232.0 → synapse-2.233.0}/synapse/datamodel.py

@@ -862,6 +862,16 @@ class Model:
         # now we can load all the forms...
         for _, mdef in mods:
 
+            # Allow props declared directly on ctors to become forms...
+            for name, ctor, opts, info in mdef.get('ctors', ()):
+                if (props := info.get('props')) is not None:
+                    self.addForm(name, {}, props, checks=False)
+
+            # Allow props declared directly on types to become forms...
+            for typename, (basename, typeopts), typeinfo in mdef.get('types', ()):
+                if (props := typeinfo.get('props')) is not None:
+                    self.addForm(typename, {}, props, checks=False)
+
             for formname, forminfo, propdefs in mdef.get('forms', ()):
                 self.addForm(formname, forminfo, propdefs, checks=False)
 

{synapse-2.232.0 → synapse-2.233.0}/synapse/lib/cell.py

@@ -55,6 +55,7 @@ import synapse.lib.urlhelp as s_urlhelp
 import synapse.lib.version as s_version
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.thisplat as s_thisplat
+import synapse.lib.processpool as s_processpool
 
 import synapse.lib.crypto.passwd as s_passwd
 
@@ -4465,7 +4466,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             logger.exception(f'Error while bootstrapping cell config.')
             raise
 
-        s_coro.set_pool_logging(logger, logconf=conf['_log_conf'])
+        s_processpool.set_pool_logging(logger, logconf=conf['_log_conf'])
 
         try:
             cell = await cls.anit(opts.dirn, conf=conf)

synapse-2.233.0/synapse/lib/coro.py

@@ -0,0 +1,219 @@
+'''
+Async/Coroutine related utilities.
+'''
+import asyncio
+import inspect
+import logging
+import functools
+import contextlib
+
+logger = logging.getLogger(__name__)
+
+import synapse.glob as s_glob
+import synapse.common as s_common
+
+def iscoro(item):
+    return inspect.iscoroutine(item)
+
+async def agen(item):
+    '''
+    Wrap an async_generator *or* generator in an async_generator.
+
+    Notes:
+        Do not use this for a synchronous generator which would cause
+        non-blocking IO; otherwise that IO will block the ioloop.
+    '''
+    if getattr(item, '__aiter__', None) is not None:
+        async with contextlib.aclosing(item) as agen:
+            async for x in agen:
+                yield x
+        return
+
+    for x in item:
+        yield x
+
+async def chunks(genr, size=100):
+
+    retn = []
+    async for item in genr:
+
+        retn.append(item)
+
+        if len(retn) == size:
+            yield retn
+            retn = []
+
+    if retn:
+        yield retn
+
+async def pause(genr, iterations=10):
+    idx = 0
+
+    async for out in agen(genr):
+        yield out
+        idx += 1
+
+        if idx % iterations == 0:
+            await asyncio.sleep(0)
+
+    return
+
+def executor(func, *args, **kwargs):
+    '''
+    Execute a non-coroutine function in the ioloop executor pool.
+
+    Args:
+        func: Function to execute.
+        *args: Args for the function.
+        **kwargs: Kwargs for the function.
+
+    Examples:
+
+        Execute a blocking API call in the executor pool::
+
+            import requests
+
+            def block(url, params=None):
+                return requests.get(url, params=params).json()
+
+            fut = s_coro.executor(block, 'http://some.tld/thign')
+            resp = await fut
+
+    Returns:
+        asyncio.Future: An asyncio future.
+    '''
+
+    def real():
+        return func(*args, **kwargs)
+
+    return asyncio.get_running_loop().run_in_executor(None, real)
+
+class Event(asyncio.Event):
+
+    async def timewait(self, timeout=None):
+
+        if timeout is None:
+            await self.wait()
+            return True
+
+        try:
+            await s_common.wait_for(self.wait(), timeout)
+        except asyncio.TimeoutError:
+            return False
+
+        return True
+
+async def event_wait(event: asyncio.Event, timeout=None):
+    '''
+    Wait on an an asyncio event with an optional timeout
+
+    Returns:
+        true if the event got set, False if timed out
+    '''
+    if timeout is None:
+        await event.wait()
+        return True
+
+    try:
+        await s_common.wait_for(event.wait(), timeout)
+    except asyncio.TimeoutError:
+        return False
+    return True
+
+async def waittask(task, timeout=None):
+    '''
+    Await a task without cancelling it when you time out.
+
+    Returns:
+        boolean: True if the task completed before the timeout.
+    '''
+    futu = asyncio.get_running_loop().create_future()
+    task.add_done_callback(futu.set_result)
+    try:
+        await s_common.wait_for(futu, timeout=timeout)
+        return True
+    except asyncio.TimeoutError:
+        return False
+    finally:
+        task.remove_done_callback(futu.set_result)
+
+async def ornot(func, *args, **kwargs):
+    '''
+    Calls func and awaits it if a returns a coroutine.
+
+    Note:
+        This is useful for implementing a function that might take a telepath proxy object or a local object, and you
+        must call a non-async method on that object.
+
+        This is also useful when calling a callback that might either be a coroutine function or a regular function.
+    Usage:
+        ok = await s_coro.ornot(maybeproxy.allowed, 'path')
+    '''
+
+    retn = func(*args, **kwargs)
+    if iscoro(retn):
+        return await retn
+    return retn
+
+bgtasks = set()
+def create_task(coro):
+
+    task = asyncio.get_running_loop().create_task(coro)
+    bgtasks.add(task)
+
+    def done(t):
+        bgtasks.remove(t)
+
+    task.add_done_callback(done)
+
+    return task
+
+async def await_bg_tasks(timeout=None):
+
+    if not bgtasks:
+        return []
+
+    coro = asyncio.gather(*tuple(bgtasks), return_exceptions=True)
+    try:
+        return await s_common.wait_for(coro, timeout)
+    except (asyncio.CancelledError, asyncio.TimeoutError):
+        return []
+
+class GenrHelp:
+
+    def __init__(self, genr):
+        assert genr is not None
+        self.genr = genr
+
+    def __aiter__(self):
+        return self.genr
+
+    def __iter__(self):
+
+        try:
+
+            while True:
+                item = s_glob.sync(self.genr.__anext__())
+                yield item
+
+        except StopAsyncIteration:
+            return
+
+        except GeneratorExit:
+            # Raised if a synchronous consumer exited an iterator early.
+            # Signal the generator to close down.
+            s_glob.sync(self.genr.aclose())
+            raise
+
+    async def spin(self):
+        async for x in self.genr:
+            pass
+
+    async def list(self):
+        return [x async for x in self.genr]
+
+def genrhelp(f):
+    @functools.wraps(f)
+    def func(*args, **kwargs):
+        return GenrHelp(f(*args, **kwargs))
+    return func

{synapse-2.232.0 → synapse-2.233.0}/synapse/lib/parser.py

@@ -1,5 +1,4 @@
 import ast
-import hashlib
 import collections
 
 import lark  # type: ignore
@@ -10,8 +9,8 @@ import synapse.data as s_data
 import synapse.common as s_common
 
 import synapse.lib.ast as s_ast
-import synapse.lib.coro as s_coro
 import synapse.lib.cache as s_cache
+import synapse.lib.processpool as s_processpool
 
 # TL;DR:  *rules* are the internal nodes of an abstract syntax tree (AST), *terminals* are the leaves
 
@@ -628,10 +627,10 @@ def parseEval(text):
     return Parser(text).eval()
 
 async def _forkedParseQuery(args):
-    return await s_coro._parserforked(parseQuery, args[0], mode=args[1])
+    return await s_processpool._parserforked(parseQuery, args[0], mode=args[1])
 
 async def _forkedParseEval(text):
-    return await s_coro._parserforked(parseEval, text)
+    return await s_processpool._parserforked(parseEval, text)
 
 evalcache = s_cache.FixedCache(_forkedParseEval, size=100)
 querycache = s_cache.FixedCache(_forkedParseQuery, size=100)

synapse-2.233.0/synapse/lib/process.py

@@ -0,0 +1,90 @@
+'''
+Process spawning utilities.
+'''
+import queue
+import asyncio
+import logging
+import multiprocessing
+
+logger = logging.getLogger(__name__)
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+
+import synapse.lib.coro as s_coro
+
+def _exectodo(que, todo, logconf):
+    # This is a new process: configure logging
+    s_common.setlogging(logger, **logconf)
+    func, args, kwargs = todo
+    try:
+        ret = func(*args, **kwargs)
+        que.put(ret)
+    except s_exc.SynErr as e:
+        logger.exception(f'Error executing spawn function {func}')
+        que.put(e)
+    except Exception as e:
+        # exceptions could be non-pickleable so wrap in SynErr
+        logger.exception(f'Error executing spawn function {func}')
+        name, info = s_common.err(e)
+        mesg = f'Error executing spawn function: {name}: {info.get("mesg")}'
+        exc = s_exc.SynErr(mesg=mesg, name=name, info=info)
+        que.put(exc)
+
+async def spawn(todo, timeout=None, ctx=None, log_conf=None):
+    '''
+    Run a todo (func, args, kwargs) tuple in a multiprocessing subprocess.
+
+    Args:
+        todo (tuple): A tuple of function, ``*args``, and ``**kwargs``.
+        timeout (int): The timeout to wait for the todo function to finish.
+        ctx (multiprocess.Context): A optional multiprocessing context object.
+        log_conf (dict): An optional logging configuration for the spawned process.
+
+    Notes:
+        The contents of the todo tuple must be able to be pickled for execution.
+        This means that locally bound functions are not eligible targets for spawn.
+
+    Returns:
+        The return value of executing the todo function.
+    '''
+    if ctx is None:
+        ctx = multiprocessing.get_context('spawn')
+    if log_conf is None:
+        log_conf = {}
+
+    que = ctx.Queue()
+    proc = ctx.Process(target=_exectodo,
+                       args=(que, todo, log_conf))
+
+    def execspawn():
+
+        proc.start()
+
+        while True:
+            try:
+                # we have to block/wait on the queue because the sender
+                # could need to stream the return value in multiple chunks
+                retn = que.get(timeout=1)
+                # now that we've retrieved the response, it should have exited.
+                proc.join()
+                return retn
+            except queue.Empty:
+                if not proc.is_alive():
+                    proc.join()
+                    mesg = f'Spawned process {proc} exited for {todo[0]} without a result.'
+                    raise s_exc.SpawnExit(mesg=mesg, code=proc.exitcode)
+
+    try:
+
+        coro = s_coro.executor(execspawn)
+
+        retn = await s_common.wait_for(coro, timeout=timeout)
+        if isinstance(retn, Exception):
+            raise retn
+
+        return retn
+
+    except (asyncio.CancelledError, asyncio.TimeoutError):
+        proc.terminate()
+        raise

synapse-2.233.0/synapse/lib/processpool.py

@@ -0,0 +1,122 @@
+'''
+Process pool related utilities.
+
+Importing this module has the side effect of creating a forkserver and should be done
+early in the creation of a process which needs it.
+'''
+import os
+import atexit
+import asyncio
+import logging
+import multiprocessing
+import concurrent.futures
+
+logger = logging.getLogger(__name__)
+
+import synapse.exc as s_exc
+import synapse.common as s_common
+import synapse.lib.process as s_process
+
+forkpool = None
+forkpool_sema = None
+max_workers = None
+def_max_workers = 8
+reserved_workers = 2
+if multiprocessing.current_process().name == 'MainProcess':
+    # only create the forkpools in the MainProcess...
+    try:
+        mpctx = multiprocessing.get_context('forkserver')
+        max_workers = int(os.getenv('SYN_FORKED_WORKERS', 0)) or max(def_max_workers, os.cpu_count() or def_max_workers)
+        forkpool = concurrent.futures.ProcessPoolExecutor(mp_context=mpctx, max_workers=max_workers)
+        atexit.register(forkpool.shutdown)
+        forkpool_sema = asyncio.Semaphore(max(1, max_workers - reserved_workers))
+    except OSError as e:  # pragma: no cover
+        max_workers = None
+        logger.warning(f'Failed to init forkserver pool, fallback enabled: {e}', exc_info=True)
+
+def _runtodo(todo):  # pragma: no cover
+    return todo[0](*todo[1], **todo[2])
+
+def _init_pool_worker(logger_, logconf):  # prama: no cover
+    s_common.setlogging(logger_, **logconf)
+    p = multiprocessing.current_process()
+    logger.debug(f'Initialized new forkserver pool worker: name={p.name} pid={p.ident}')
+
+_pool_logconf = None
+def set_pool_logging(logger_, logconf):
+    # This must be called before any calls to forked() and _parserforked()
+    global _pool_logconf
+    _pool_logconf = logconf
+    todo = s_common.todo(_init_pool_worker, logger_, logconf)
+    if forkpool is not None:
+        forkpool._initializer = _runtodo
+        forkpool._initargs = (todo,)
+
+async def forked(func, *args, **kwargs):
+    '''
+    Execute a target function in the shared forked process pool
+    and fallback to running in a spawned process if the pool is unavailable.
+
+    Args:
+        func: The target function.
+        *args: Function positional arguments.
+        **kwargs: Function keyword arguments.
+
+    Returns:
+        The target function return.
+    '''
+    todo = (func, args, kwargs)
+
+    if forkpool is not None:
+        try:
+            return await asyncio.get_running_loop().run_in_executor(forkpool, _runtodo, todo)
+        except concurrent.futures.process.BrokenProcessPool as e:  # pragma: no cover
+            logger.exception(f'Shared forkserver pool is broken, fallback enabled: {func}')
+
+    logger.debug(f'Forkserver pool using spawn fallback: {func}')
+    return await s_process.spawn(todo, log_conf=_pool_logconf)
+
+async def semafork(func, *args, **kwargs):
+    '''
+    Execute a target function in the shared forked process pool
+    gated by a semaphore to ensure there are workers reserved for the Storm parser.
+
+    Args:
+        func: The target function.
+        *args: Function positional arguments.
+        **kwargs: Function keyword arguments.
+
+    Returns:
+        The target function return.
+    '''
+    if forkpool_sema is None:
+        return await forked(func, *args, **kwargs)
+
+    async with forkpool_sema:
+        return await forked(func, *args, **kwargs)
+
+async def _parserforked(func, *args, **kwargs):
+    '''
+    Execute a target function in the shared forked process pool
+    and fallback to running in the default executor if the pool is unavailable.
+
+    NOTE: This function is intended to only be used by the Storm parser
+
+    Args:
+        func: The target function.
+        *args: Function positional arguments.
+        **kwargs: Function keyword arguments.
+
+    Returns:
+        The target function return.
+
+    Raises:
+        The function may raise from the target function, or raise an s_exc.FatalErr in the event of a broken forked
+        process pool. The fatalerr represents an unrecoverable application state.
+    '''
+    todo = (func, args, kwargs)
+    try:
+        return await asyncio.get_running_loop().run_in_executor(forkpool, _runtodo, todo)
+    except concurrent.futures.process.BrokenProcessPool as e:  # pragma: no cover
+        logger.exception(f'Fatal error executing forked task: {func} {args} {kwargs}')
+        raise s_exc.FatalErr(mesg=f'Fatal error encountered: {e}') from None

{synapse-2.232.0 → synapse-2.233.0}/synapse/lib/scrape.py

@@ -1,5 +1,4 @@
 import string
-import asyncio
 import logging
 import pathlib
 import functools
@@ -14,9 +13,7 @@ import synapse.data as s_data
 import synapse.common as s_common
 
 import synapse.lib.chop as s_chop
-import synapse.lib.coro as s_coro
-import synapse.lib.link as s_link
-import synapse.lib.msgpack as s_msgpack
+import synapse.lib.processpool as s_processpool
 
 import synapse.lib.crypto.coin as s_coin
 
@@ -575,7 +572,7 @@ async def genMatchesAsync(text: str, regx: regex.Regex, opts: dict):
     Yields:
         dict: A dictionary of match results.
     '''
-    matches = await s_coro.semafork(_genMatchList, text, regx, opts)
+    matches = await s_processpool.semafork(_genMatchList, text, regx, opts)
     for info in matches:
         yield info
 
@@ -687,7 +684,7 @@ async def contextScrapeAsync(text, form=None, refang=True, first=False):
     Returns:
         (dict): Yield info dicts of results.
     '''
-    matches = await s_coro.semafork(_contextScrapeList, text, form=form, refang=refang, first=first)
+    matches = await s_processpool.semafork(_contextScrapeList, text, form=form, refang=refang, first=first)
     for info in matches:
         yield info
 
@@ -704,6 +701,6 @@ async def scrapeAsync(text, ptype=None, refang=True, first=False):
     Returns:
         (str, object): Yield tuples of node ndef values.
     '''
-    matches = await s_coro.semafork(_contextScrapeList, text, form=ptype, refang=refang, first=first)
+    matches = await s_processpool.semafork(_contextScrapeList, text, form=ptype, refang=refang, first=first)
     for info in matches:
         yield info.get('form'), info.get('valu')

{synapse-2.232.0 → synapse-2.233.0}/synapse/lib/storm.py

@@ -1792,9 +1792,9 @@ class Runtime(s_base.Base):
                 async for item in nodegenr:
                     yield item
 
-        except RecursionError:
+        except RecursionError as e:
             mesg = 'Maximum Storm pipeline depth exceeded.'
-            raise s_exc.RecursionLimitHit(mesg=mesg, query=self.query.text) from None
+            raise s_exc.RecursionLimitHit(mesg=mesg, query=self.query.text) from e
 
     async def _snapFromOpts(self, opts):
 

{synapse-2.232.0 → synapse-2.233.0}/synapse/lib/stormlib/json.py

@@ -4,10 +4,10 @@ import logging
 
 import synapse.exc as s_exc
 
-import synapse.lib.coro as s_coro
 import synapse.lib.json as s_json
 import synapse.lib.config as s_config
 import synapse.lib.stormtypes as s_stormtypes
+import synapse.lib.processpool as s_processpool
 
 logger = logging.getLogger(__name__)
 
@@ -71,7 +71,7 @@ class JsonSchema(s_stormtypes.StormType):
         item = await s_stormtypes.toprim(item)
 
         try:
-            result = await s_coro.semafork(runJsSchema, self.schema, item, use_default=self.use_default)
+            result = await s_processpool.semafork(runJsSchema, self.schema, item, use_default=self.use_default)
         except s_exc.SchemaViolation as e:
             return False, {'mesg': e.get('mesg')}
         else:
@@ -138,7 +138,7 @@ class JsonLib(s_stormtypes.Lib):
         use_default = await s_stormtypes.tobool(use_default)
         # We have to ensure that we have a valid schema for making the object.
         try:
-            await s_coro.semafork(compileJsSchema, schema, use_default=use_default)
+            await s_processpool.semafork(compileJsSchema, schema, use_default=use_default)
         except asyncio.CancelledError:  # pragma: no cover
             raise
         except Exception as e:

{synapse-2.232.0 → synapse-2.233.0}/synapse/lib/stormlib/mime.py

@@ -1,7 +1,7 @@
 import bs4
 
-import synapse.lib.coro as s_coro
 import synapse.lib.stormtypes as s_stormtypes
+import synapse.lib.processpool as s_processpool
 
 def htmlToText(html, separator='\n', strip=True):
     soup = bs4.BeautifulSoup(html, 'html5lib')
@@ -42,4 +42,4 @@ class LibMimeHtml(s_stormtypes.Lib):
         if separator is None:
             separator = ''
 
-        return await s_coro.semafork(htmlToText, html, separator, strip)
+        return await s_processpool.semafork(htmlToText, html, separator, strip)

{synapse-2.232.0 → synapse-2.233.0}/synapse/lib/stormlib/scrape.py

@@ -5,10 +5,10 @@ import regex
 
 import synapse.common as s_common
 
-import synapse.lib.coro as s_coro
 import synapse.lib.scrape as s_scrape
 import synapse.lib.spooled as s_spooled
 import synapse.lib.stormtypes as s_stormtypes
+import synapse.lib.processpool as s_processpool
 
 logger = logging.getLogger(__name__)
 
@@ -156,7 +156,7 @@ class LibScrape(s_stormtypes.Lib):
         if fangs:
             _fangs = {src: dst for (src, dst) in fangs}
             _fangre = s_scrape.genFangRegex(_fangs)
-            scrape_text, offsets = await s_coro.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
+            scrape_text, offsets = await s_processpool.semafork(s_scrape.refang_text2, text, re=_fangre, fangs=_fangs)
 
         async for info in s_scrape.genMatchesAsync(scrape_text, regx, opts=opts):
             valu = info.pop('valu')