synapse 2.176.0__py311-none-any.whl → 2.178.0__py311-none-any.whl

synapse/tests/test_tools_changelog.py

@@ -0,0 +1,196 @@
+import synapse.tests.utils as s_test_utils
+
+import synapse.tools.changelog as s_t_changelog
+
+class ChangelogToolTest(s_test_utils.SynTest):
+
+    async def test_model_diff(self):
+        outp = self.getTestOutp()
+        old_fp = self.getTestFilePath('changelog', 'model_2.176.0_16ee721a6b7221344eaf946c3ab4602dda546b1a.yaml.gz')
+        new_fp = self.getTestFilePath('changelog', 'model_2.176.0_2a25c58bbd344716cd7cbc3f4304d8925b0f4ef2.yaml.gz')
+        oldmodel = s_t_changelog._getModelFile(old_fp)
+        newmodel = s_t_changelog._getModelFile(new_fp)
+
+        differ = s_t_changelog.ModelDiffer(newmodel.get('model'), oldmodel.get('model'))
+
+        self.eq(differ.cur_type2iface['it:host'], ['inet:service:object', 'inet:service:base'])
+        self.eq(differ.cur_type2iface['mat:type'], ['meta:taxonomy'])
+        self.eq(differ.cur_iface_to_allifaces['it:host:activity'], ['it:host:activity'])
+        self.eq(differ.cur_iface_to_allifaces['file:mime:msoffice'], ['file:mime:msoffice', 'file:mime:meta'])
+        changes = differ.diffModl(outp)
+
+        self.len(4, changes.get('edges').get('new_edges'))
+
+        form_chng = changes.get('forms')
+        self.isin('it:beeper:thingy', form_chng.get('new_forms'))
+        self.notin('it:beeper:name', form_chng.get('new_forms'))
+        updt_frms = form_chng.get('updated_forms')
+        self.isin('new_properties', updt_frms.get('file:mime:pe:resource'))
+        self.isin('new_properties_no_interfaces', updt_frms.get('file:mime:pe:resource'))
+        self.isin('new_properties', updt_frms.get('inet:http:request'))
+        self.notin('new_properties_no_interfaces', updt_frms.get('inet:http:request'))
+        self.isin('updated_properties', updt_frms.get('file:mime:macho:section'))
+        self.isin('deprecated_properties', updt_frms.get('file:mime:lnk'))
+        self.isin('deprecated_properties_no_interfaces', updt_frms.get('file:mime:lnk'))
+        self.isin('deprecated_properties', updt_frms.get('inet:http:request'))
+        self.notin('deprecated_properties_no_interfaces', updt_frms.get('inet:http:request'))
+        self.isin('new_properties', updt_frms.get('file:mime:lnk'))
+        self.isin('new_properties_no_interfaces', updt_frms.get('file:mime:lnk'))
+        self.isin('updated_properties', updt_frms.get('file:mime:lnk'))
+        self.isin('updated_properties_no_interfaces', updt_frms.get('file:mime:lnk'))
+        self.isin('updated_properties', updt_frms.get('inet:service:access'))
+        self.notin('updated_properties_no_interfaces', updt_frms.get('inet:service:access'))
+
+        self.isin('it:host:beeper', changes.get('interfaces').get('new_interfaces'))
+        updt_iface = changes.get('interfaces').get('updated_interfaces')
+        self.isin('deprecated_properties', updt_iface.get('it:host:activity'))
+        self.isin('new_properties', updt_iface.get('it:host:activity'))
+        self.isin('updated_properties', updt_iface.get('inet:service:base'))
+
+        self.eq(changes.get('tagprops'), {})
+
+        self.isin('it:reveng:function', changes.get('types').get('deprecated_types'))
+        self.isin('inet:port', changes.get('types').get('deprecated_types'))
+        self.isin('it:beeper:thingy', changes.get('types').get('new_types'))
+        self.isin('it:beeper:name', changes.get('types').get('new_types'))
+        uptd_types = changes.get('types').get('updated_types')
+        self.isin('updated_opts', uptd_types.get('it:query'))
+        self.isin('updated_interfaces', uptd_types.get('it:reveng:impfunc'))
+
+        rst = s_t_changelog._gen_model_rst('v2.177.0', 'userguide_model_v2_177_0',
+                                           changes, newmodel.get('model'), outp)
+        text = rst.getRstText()
+        self.isin('v2.177.0 Model Updates', text)
+        self.isin('''**************
+New Interfaces
+**************
+
+``it:host:beeper``
+  Properties common to instances of beepers.''', text)
+        self.isin('''*********
+New Types
+*********
+
+``it:beeper:name``
+  The friendly name of a beeper.
+''', text)
+        self.isin('''*********
+New Forms
+*********
+
+``it:beeper:thingy``
+  A beeper thingy.
+''', text)
+        self.isin('''**************
+New Properties
+**************
+
+``file:mime:lnk``
+  The form had the following properties added to it:
+
+
+  ``driveserial``
+    The drive serial number of the volume the link target is stored on.
+
+
+  ``iconindex``
+    A resource index for an icon within an icon location.
+''', text)
+        self.isin('''``inet:http:request``
+  The form had the following property added to it:
+
+  ``beep``
+    The time that the host went beep.
+''', text)
+        # Some updates will require manual intervention to rewrite.
+        self.isin('''******************
+Updated Interfaces
+******************
+
+``inet:service:base``
+  The property ``id`` has been modified from ['str', {'strip': True}] to ['str',
+  {'lower': True, 'strip': True}].
+
+
+``it:host:activity``
+  The interface property ``time`` has been deprecated.
+
+
+  The property ``beep`` has been added to the interface.
+''', text)
+        self.isin('''*************
+Updated Types
+*************
+
+``it:query``
+  The type has been modified from {'enums': None, 'globsuffix': False, 'lower':
+  False, 'onespace': False, 'regex': None, 'replace': [], 'strip': True} to
+  {'enums': None, 'globsuffix': False, 'lower': True, 'onespace': False,
+  'regex': None, 'replace': [], 'strip': True}.
+
+
+``it:reveng:impfunc``
+  The type interface has been modified from None to ['it:host:beeper'].
+''', text)
+        self.isin('''******************
+Updated Properties
+******************
+
+``file:mime:lnk``
+  The form had the following property updated:
+
+
+    The property ``entry:icon`` has been modified from ['file:path', {}] to
+    ['time', {}].
+''', text)
+        self.isin('''***********
+Light Edges
+***********
+
+``jenkies``
+    When used with a ``it:prod:soft`` node, the edge indicates The software uses
+    the jenkies technique.
+
+
+``loves``
+    The source node loves the target node.
+
+
+``sneaky``
+    When used with a ``ou:technique`` target node, the edge indicates The
+    technique referred to is sneakily used.
+
+
+``zoinks``
+    When used with a ``it:prod:soft`` and an ``ou:technique`` node, the edge
+    indicates The software uses the zoinks technique.
+''', text)
+        self.isin('''****************
+Deprecated Types
+****************
+
+The following types have been marked as deprecated:
+
+
+* ``inet:port``
+
+''', text)
+        self.isin('''****************
+Deprecated Types
+****************
+
+The following forms have been marked as deprecated:
+
+
+* ``it:reveng:function``
+''', text)
+        self.isin('''*********************
+Deprecated Properties
+*********************
+
+``file:mime:lnk``
+  The form had the following property deprecated:
+
+  ``target:attrs``
+    The attributes of the target file according to the LNK header.
+''', text)

synapse/tests/test_tools_healthcheck.py

@@ -65,11 +65,12 @@ class HealthcheckTest(s_t_utils.SynTest):
             m = 'Synapse error encountered.'
             self.eq(resp.get('components')[0].get('mesg'), m)
 
-            # dont do this in prod...
+            visi = await core.auth.addUser('visi')
+            await visi.setPasswd('secret')
+
             logger.info('Checking without perms')
-            await root.setAdmin(False)
             outp.clear()
-            retn = await s_t_healthcheck.main(['-c', curl, '-t', '0.2'], outp)
+            retn = await s_t_healthcheck.main(['-c', f'tcp://visi:secret@127.0.0.1:{port}/cortex', '-t', '0.2'], outp)
             self.eq(retn, 1)
             resp = json.loads(str(outp))
             self.eq(resp.get('components')[0].get('name'), 'error')

synapse/tests/utils.py

@@ -677,7 +677,7 @@ class TstEnv:
 
 class TstOutPut(s_output.OutPutStr):
 
-    def expect(self, substr, throw=True):
+    def expect(self, substr, throw=True, whitespace=True):
         '''
         Check if a string is present in the messages captured by the OutPutStr object.
 
@@ -689,6 +689,11 @@ class TstOutPut(s_output.OutPutStr):
             bool: True if the string is present; False if the string is not present and throw is False.
         '''
         outs = str(self)
+
+        if not whitespace:
+            outs = ' '.join(outs.split())
+            substr = ' '.join(outs.split())
+
         if outs.find(substr) == -1:
             if throw:
                 mesg = 'TestOutPut.expect(%s) not in %s' % (substr, outs)
@@ -1297,34 +1302,20 @@ class SynTest(unittest.TestCase):
         Returns:
             s_cortex.Cortex: A Cortex object.
         '''
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-
-        conf = copy.deepcopy(conf)
+        conf = self.getCellConf(conf=conf)
 
-        mods = conf.get('modules')
-
-        if mods is None:
-            mods = []
-            conf['modules'] = mods
+        mods = list(conf.get('modules', ()))
+        conf['modules'] = mods
 
         mods.insert(0, ('synapse.tests.utils.TestModule', {'key': 'valu'}))
 
         with self.withNexusReplay():
 
-            if dirn is not None:
+            with self.mayTestDir(dirn) as dirn:
 
                 async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
                     yield core
 
-                return
-
-            with self.getTestDir() as dirn:
-                async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
-                    yield core
-
     @contextlib.asynccontextmanager
     async def getTestCoreAndProxy(self, conf=None, dirn=None):
         '''
@@ -1342,11 +1333,7 @@ class SynTest(unittest.TestCase):
     @contextlib.asynccontextmanager
     async def getTestJsonStor(self, dirn=None, conf=None):
 
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-        conf = copy.deepcopy(conf)
+        conf = self.getCellConf(conf=conf)
 
         with self.withNexusReplay():
             if dirn is not None:
@@ -1367,20 +1354,10 @@ class SynTest(unittest.TestCase):
         Returns:
             s_cryotank.CryoCell: Test cryocell.
         '''
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-        conf = copy.deepcopy(conf)
+        conf = self.getCellConf(conf=conf)
 
         with self.withNexusReplay():
-            if dirn is not None:
-                async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
-                    yield cryo
-
-                return
-
-            with self.getTestDir() as dirn:
+            with self.mayTestDir(dirn) as dirn:
                 async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
                     yield cryo
 
@@ -1409,26 +1386,13 @@ class SynTest(unittest.TestCase):
             s_certdir.delCertPath(certpath)
 
     @contextlib.asynccontextmanager
-    async def getTestCell(self, ctor, conf=None, dirn=None):
+    async def getTestCell(self, ctor=s_cell.Cell, conf=None, dirn=None):
         '''
         Get a test Cell.
         '''
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-
-        conf = copy.deepcopy(conf)
-
+        conf = self.getCellConf(conf=conf)
         with self.withNexusReplay():
-            if dirn is not None:
-
-                async with await ctor.anit(dirn, conf=conf) as cell:
-                    yield cell
-
-                return
-
-            with self.getTestDir() as dirn:
+            with self.mayTestDir(dirn) as dirn:
                 async with await ctor.anit(dirn, conf=conf) as cell:
                     yield cell
 
@@ -1451,62 +1415,77 @@ class SynTest(unittest.TestCase):
 
                 yield core, prox, testsvc
 
+    @contextlib.contextmanager
+    def mayTestDir(self, dirn: str | None) -> contextlib.AbstractContextManager[str, None, None]:
+        '''
+        Convenience method to make a temporary directory or use an existing one.
+
+        Args:
+            dirn: Directory to use, or None.
+
+        Returns:
+            The directory to use.
+        '''
+        if dirn is not None:
+            yield dirn
+            return
+
+        with self.getTestDir() as dirn:
+            yield dirn
+
     @contextlib.asynccontextmanager
-    async def getTestAha(self, conf=None, dirn=None):
+    async def getTestAha(self, conf=None, dirn=None, ctor=None):
 
         if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-        conf = copy.deepcopy(conf)
+            conf = {}
+
+        if ctor is None:
+            ctor = s_aha.AhaCell.anit
+
+        conf = s_msgpack.deepcopy(conf)
+
+        hasdmonlisn = 'dmon:listen' in conf
+        hasprovlisn = 'provision:listen' in conf
+
+        network = conf.setdefault('aha:network', 'synapse')
+        hostname = conf.setdefault('dns:name', '00.aha.loop.vertex.link')
+
+        if hostname:
+            conf.setdefault('provision:listen', f'ssl://0.0.0.0:0?hostname={hostname}')
+            conf.setdefault('dmon:listen', f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}')
+
+        conf.setdefault('health:sysctl:checks', False)
 
         with self.withNexusReplay():
-            if dirn:
-                async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
-                    yield aha
-            else:
-                with self.getTestDir() as dirn:
-                    async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
-                        yield aha
 
-    @contextlib.asynccontextmanager
-    async def getTestAhaProv(self, conf=None, dirn=None):
-        '''
-        Get an Aha cell that is configured for provisioning on aha.loop.vertex.link.
+            with self.mayTestDir(dirn) as dirn:
 
-        Args:
-            conf: Optional configuration information for the Aha cell.
-            dirn: Optional path to create the Aha cell in.
+                if conf.get('aha:network') == 'synapse':
+                    dstpath = os.path.join(dirn, 'certs')
+                    if not os.path.isdir(dstpath):
+                        srcpath = self.getTestFilePath('aha/certs')
+                        shutil.copytree(srcpath, os.path.join(dirn, 'certs'))
 
-        Returns:
-            s_aha.AhaCell: The provisioned Aha cell.
-        '''
-        bconf = {
-            'aha:name': 'aha',
-            'aha:network': 'loop.vertex.link',
-            'provision:listen': 'ssl://aha.loop.vertex.link:0'
-        }
+                async with await ctor(dirn, conf=conf) as aha:
 
-        if conf is None:
-            conf = bconf
-        else:
-            for k, v in bconf.items():
-                conf.setdefault(k, v)
+                    mods = {}
+                    if not hasdmonlisn and hostname:
+                        mods['dmon:listen'] = f'ssl://0.0.0.0:{aha.sockaddr[1]}?hostname={hostname}&ca={network}'
 
-        name = conf.get('aha:name')
-        netw = conf.get('aha:network')
-        dnsname = f'{name}.{netw}'
+                    if not hasprovlisn and hostname:
+                        mods['provision:listen'] = f'ssl://0.0.0.0:{aha.provaddr[1]}?hostname={hostname}'
 
-        async with self.getTestAha(conf=conf, dirn=dirn) as aha:
-            addr, port = aha.provdmon.addr
-            # update the config to reflect the dynamically bound port
-            aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}'
+                    if mods:
+                        aha.modCellConf(mods)
 
-            # do this config ex-post-facto due to port binding...
-            host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}')
-            aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',)
+                    yield aha
 
-            yield aha
+    def getCellConf(self, conf=None):
+        if conf is None:
+            conf = {}
+        conf = s_msgpack.deepcopy(conf)
+        conf.setdefault('health:sysctl:checks', False)
+        return conf
 
     @contextlib.asynccontextmanager
     async def addSvcToAha(self, aha, svcname, ctor,
@@ -1528,36 +1507,19 @@ class SynTest(unittest.TestCase):
             The config data for the cell is pushed into dirn/cell.yaml.
             The cells are created with the ``ctor.anit()`` function.
         '''
+        ahanetw = aha.conf.req('aha:network')
+        svcfull = f'{svcname}.{ahanetw}'
         onetime = await aha.addAhaSvcProv(svcname, provinfo=provinfo)
 
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-
+        conf = self.getCellConf(conf=conf)
         conf['aha:provision'] = onetime
 
-        logger.info(f'Adding {svcname}')
-
-        n = 1  # a simple svcname like "foo"
-        if len(svcname.split('.')) > 1:
-            n = 2  # A replica name like 00.foo
-        if provinfo and 'mirror' in provinfo:
-            n = 1  # A mirror like 01.foo with mirror: foo
-
-        waiter = aha.waiter(n, 'aha:svcadd')
-
-        if dirn:
+        waiter = aha.waiter(1, f'aha:svcadd:{svcfull}')
+        with self.mayTestDir(dirn) as dirn:
             s_common.yamlsave(conf, dirn, 'cell.yaml')
             async with await ctor.anit(dirn) as svc:
-                self.ge(len(await waiter.wait(timeout=12)), n)
+                self.true(await waiter.wait(timeout=12))
                 yield svc
-        else:
-            with self.getTestDir() as dirn:
-                s_common.yamlsave(conf, dirn, 'cell.yaml')
-                async with await ctor.anit(dirn) as svc:
-                    self.ge(len(await waiter.wait(timeout=12)), n)
-                    yield svc
 
     async def addSvcToCore(self, svc, core, svcname='svc'):
         '''
@@ -1592,7 +1554,7 @@ class SynTest(unittest.TestCase):
         return s_telepath.openurl(f'tcp:///{name}', **kwargs)
 
     @contextlib.contextmanager
-    def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None):
+    def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None) -> contextlib.AbstractContextManager[str, None, None]:
         '''
         Get a temporary directory for test purposes.
         This destroys the directory afterwards.
@@ -1785,7 +1747,7 @@ class SynTest(unittest.TestCase):
                     await stream.wait(timeout=10)
 
                 data = stream.getvalue()
-                raw_mesgs = [m for m in data.split('\\n') if m]
+                raw_mesgs = [m for m in data.split('\n') if m]
                 msgs = [json.loads(m) for m in raw_mesgs]
                 # Do something with messages
 
@@ -2058,6 +2020,7 @@ class SynTest(unittest.TestCase):
         Assert X is not None
         '''
         self.assertIsNotNone(x, msg=msg)
+        return x
 
     def none(self, x, msg=None):
         '''
@@ -2154,7 +2117,7 @@ class SynTest(unittest.TestCase):
             count += 1
         self.eq(x, count, msg=msg)
 
-    def stormIsInPrint(self, mesg, mesgs, deguid=False):
+    def stormIsInPrint(self, mesg, mesgs, deguid=False, whitespace=True):
         '''
         Check if a string is present in all of the print messages from a stream of storm messages.
 
@@ -2166,6 +2129,9 @@ class SynTest(unittest.TestCase):
             mesg = deguidify(mesg)
 
         print_str = '\n'.join([m[1].get('mesg') for m in mesgs if m[0] == 'print'])
+        if not whitespace:
+            mesg = ' '.join(mesg.split())
+            print_str = ' '.join(print_str.split())
 
         if deguid:
             print_str = deguidify(print_str)

synapse/tools/aha/clone.py

@@ -0,0 +1,50 @@
+import sys
+import asyncio
+import argparse
+
+import synapse.exc as s_exc
+import synapse.telepath as s_telepath
+
+import synapse.lib.output as s_output
+
+descr = '''
+Generate a new clone URL to deploy an AHA mirror.
+
+Examples:
+
+    python -m synapse.tools.aha.clone 01.aha.loop.vertex.link
+
+'''
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = argparse.ArgumentParser(prog='synapse.tools.aha.clone', description=descr)
+    pars.add_argument('dnsname', help='The DNS name of the new AHA server.')
+    pars.add_argument('--port', type=int, default=27492, help='The port that the new AHA server should listen on.')
+    pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.')
+    pars.add_argument('--only-url', help='Only output the URL upon successful execution',
+                      action='store_true', default=False)
+    opts = pars.parse_args(argv)
+
+    async with s_telepath.withTeleEnv():
+
+        try:
+            async with await s_telepath.openurl(opts.url) as aha:
+                curl = await aha.addAhaClone(opts.dnsname, port=opts.port)
+
+                if opts.only_url:
+                    outp.printf(curl)
+                else:
+                    outp.printf(f'one-time use URL: {curl}')
+                return 0
+
+        except Exception as e:
+            mesg = repr(e)
+            if isinstance(e, s_exc.SynErr):
+                mesg = e.errinfo.get('mesg', repr(e))
+
+            outp.printf(f'ERROR: {mesg}')
+            return 1
+
+if __name__ == '__main__':  # pragma: no cover
+    sys.exit(asyncio.run(main(sys.argv[1:])))

synapse/tools/aha/enroll.py

@@ -76,7 +76,8 @@ async def main(argv, outp=s_output.stdout):
                 if isinstance(ahaurls, str):
                     ahaurls = (ahaurls,)
 
-                ahaurls = set(s_telepath.modurl(ahaurls, user=ahauser))
+                certname = f'{ahauser}@{ahanetw}'
+                ahaurls = set(s_telepath.modurl(ahaurls, certname=certname))
                 servers = teleyaml.get('aha:servers')
 
                 # repack the servers so lists are tuplized like values

synapse/tools/backup.py

@@ -148,7 +148,7 @@ def txnbackup(lmdbinfo, srcdir, dstdir, skipdirs=None):
     logger.debug(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
     return
 
-def backup_lmdb(env, dstdir, txn=None):
+def backup_lmdb(env: lmdb.Environment, dstdir: str, txn=None):
 
     tick = time.time()
 
@@ -157,7 +157,7 @@ def backup_lmdb(env, dstdir, txn=None):
     env.copy(dstdir, compact=True, txn=txn)
 
     tock = time.time()
-    logger.info(f'backup took: {tock-tick:.2f} seconds')
+    logger.info(f'backup of: {env.path()} took: {tock-tick:.2f} seconds')
 
 def main(argv):
     args = parse_args(argv)