synapse 2.187.0__tar.gz → 2.187.1__tar.gz

{synapse-2.187.0/synapse.egg-info → synapse-2.187.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: synapse
-Version: 2.187.0
+Version: 2.187.1
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -19,7 +19,7 @@ Classifier: Operating System :: POSIX :: Linux
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: pyOpenSSL<25.0.0,>=24.0.0
+Requires-Dist: pyOpenSSL<24.3.0,>=24.0.0
 Requires-Dist: cryptography<44.0.0,>=43.0.1
 Requires-Dist: msgpack<1.1.0,>=1.0.5
 Requires-Dist: xxhash<3.6.0,>=1.4.4
@@ -63,6 +63,7 @@ Requires-Dist: sphinx<9.0.0,>=8.0.0; extra == "docs"
 Requires-Dist: sphinx-rtd-theme<4.0.0,>=3.0.0; extra == "docs"
 Requires-Dist: sphinx-notfound-page<2.0.0,>=1.0.4; extra == "docs"
 Requires-Dist: jinja2<4.0.0,>=3.1.4; extra == "docs"
+Dynamic: license-file
 
 Synapse
 =======

{synapse-2.187.0 → synapse-2.187.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'synapse'
-version = '2.187.0'
+version = '2.187.1'
 authors = [
     { name = 'The Vertex Project LLC', email = 'root@vertex.link'},
 ]
@@ -23,7 +23,7 @@ classifiers = [
     'Operating System :: POSIX :: Linux',
 ]
 dependencies = [
-    'pyOpenSSL>=24.0.0,<25.0.0',
+    'pyOpenSSL>=24.0.0,<24.3.0',
     'cryptography>=43.0.1,<44.0.0',
     'msgpack>=1.0.5,<1.1.0',
     'xxhash>=1.4.4,<3.6.0',

{synapse-2.187.0 → synapse-2.187.1}/synapse/lib/lmdbslab.py

@@ -1931,6 +1931,8 @@ class Scan:
         '''
         Returns if the cursor is at the value in atitem
         '''
+        if not self.dupsort:
+            return self.atitem[0] == self.curs.item()[0]
         return self.atitem == self.curs.item()
 
 class ScanKeys(Scan):

{synapse-2.187.0 → synapse-2.187.1}/synapse/lib/modelrev.py

@@ -1446,7 +1446,7 @@ class ModelMigration_0_2_31:
 
         # Pick up and classify all bad CPE nodes
         for idx, layer in enumerate(self.layers):
-            logger.debug(f'Classifying nodes in layer {idx}')
+            logger.debug('Classifying nodes in layer %s %s', idx, layer.iden)
 
             async for buid, sode in layer.getStorNodesByForm('it:sec:cpe'):
 
@@ -1457,6 +1457,7 @@ class ModelMigration_0_2_31:
                 if (v2_2 := props.get('v2_2')) is not None:
                     propvalu, stortype = v2_2
                     if not s_infotech.isValidCpe22(propvalu):
+                        logger.debug(f'Queueing invalid v2_2 value for deletion iden={s_common.ehex(buid)} valu={propvalu}')
                         await self._queueEdit(
                             layer.iden,
                             (buid, 'it:sec:cpe', (
@@ -1490,7 +1491,7 @@ class ModelMigration_0_2_31:
 
         # Pick up all related CPE node info. The majority of the work happens in this loop
         for idx, layer in enumerate(self.layers):
-            logger.debug(f'Processing nodes in layer {idx}')
+            logger.debug('Processing nodes in layer %s %s', idx, layer.iden)
 
             for buid, node in self.nodes.items():
                 await self._loadNode(layer, buid, node=node)
@@ -1532,7 +1533,7 @@ class ModelMigration_0_2_31:
             await self.todos.clear()
 
             for idx, layer in enumerate(self.layers):
-                logger.debug(f'Processing references in layer {idx}')
+                logger.debug('Processing references in layer %s %s', idx, layer.iden)
 
                 async for entry in todotmp:
                     match entry:
@@ -1613,10 +1614,25 @@ class ModelMigration_0_2_31:
                     if propvalu is not None:
                         break
 
-                newvalu, _ = form.type.norm(propvalu)
-                await self.moveNode(buid, newvalu)
+                if propvalu is None:
+                    # We didn't find a v2_2 value so remove this node
+                    await self.removeNode(buid)
+                    removed += 1
+
+                else:
+                    # We did find a v2_2 value so try to norm it and use that new value to move the node. If this fails,
+                    # remove the node.
+                    try:
+                        newvalu, _ = form.type.norm(propvalu)
 
-                migrated += 1
+                    except s_exc.BadTypeValu:
+                        logger.debug('Unexpectedly encountered invalid v2_2 prop: iden=%s valu=%s', s_common.ehex(buid), propvalu)
+                        await self.removeNode(buid)
+                        removed += 1
+
+                    else:
+                        await self.moveNode(buid, newvalu)
+                        migrated += 1
 
             elif action == 'remove':
                 newvalu = None
@@ -1627,10 +1643,17 @@ class ModelMigration_0_2_31:
                     if propvalu is None:
                         continue
 
-                    newvalu, _ = form.type.norm(propvalu)
                     # This prop is going to be the new primary value so delete the secondary prop
                     await self.editPropDel(layriden, buid, 'it:sec:cpe', 'v2_2', propvalu, stortype)
 
+                    # We did find a v2_2 value so try to norm it and use that new value to move the node. If this fails,
+                    # remove the node.
+                    try:
+                        newvalu, _ = form.type.norm(propvalu)
+                    except s_exc.BadTypeValu:
+                        logger.debug('Unexpectedly encountered invalid v2_2 prop: iden=%s valu=%s', s_common.ehex(buid), propvalu)
+                        continue
+
                     # Oh yeah! Migrate the node instead of removing it
                     await self.moveNode(buid, newvalu)
 

{synapse-2.187.0 → synapse-2.187.1}/synapse/lib/spooled.py

@@ -8,6 +8,7 @@ import synapse.lib.msgpack as s_msgpack
 import synapse.lib.lmdbslab as s_lmdbslab
 
 MAX_SPOOL_SIZE = 10000
+DEFAULT_MAPSIZE = s_const.mebibyte * 32
 
 class Spooled(s_base.Base):
     '''
@@ -48,7 +49,7 @@ class Spooled(s_base.Base):
 
         slabpath = tempfile.mkdtemp(dir=dirn, prefix='spooled_', suffix='.lmdb')
 
-        self.slab = await s_lmdbslab.Slab.anit(slabpath, map_size=s_const.mebibyte * 32)
+        self.slab = await s_lmdbslab.Slab.anit(slabpath, map_size=DEFAULT_MAPSIZE)
         if self.cell is not None:
             self.slab.addResizeCallback(self.cell.checkFreeSpace)
 

{synapse-2.187.0 → synapse-2.187.1}/synapse/lib/stormlib/model.py

@@ -1144,7 +1144,7 @@ class LibModelMigrations_0_2_31(s_stormtypes.Lib):
                 await self.runt.printf(f'    layer: {layriden}')
                 for iden, refinfo in reflist:
                     form, prop, *_ = refinfo
-                    await self.runt.printf(f'      - {form}:{prop} (iden: {iden}')
+                    await self.runt.printf(f'      - {form}:{prop} (iden: {iden})')
 
         n1edges = node['n1edges']
         n2edges = node['n2edges']

{synapse-2.187.0 → synapse-2.187.1}/synapse/lib/version.py

@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
 ##############################################################################
 # The following are touched during the release process by bumpversion.
 # Do not modify these directly.
-version = (2, 187, 0)
+version = (2, 187, 1)
 verstring = '.'.join([str(x) for x in version])
-commit = '5cdabb0eeea74eff019d8bcdc461b48bb5eb6651'
+commit = 'a75f7b990f6350f9d631f1f92ea5063542458aad'

{synapse-2.187.0 → synapse-2.187.1}/synapse/tests/test_lib_lmdbslab.py

@@ -11,6 +11,7 @@ from unittest.mock import patch
 
 import synapse.lib.base as s_base
 import synapse.lib.const as s_const
+import synapse.lib.msgpack as s_msgpack
 import synapse.lib.lmdbslab as s_lmdbslab
 import synapse.lib.thisplat as s_thisplat
 
@@ -1597,6 +1598,35 @@ class LmdbSlabTest(s_t_utils.SynTest):
                 subkv2 = subkv1.getSubKeyVal('pref2')
                 self.eq(list(subkv2.keys()), ['wow'])
 
+    async def test_lmdbslab_scan_grow(self):
+        with self.getTestDir() as dirn:
+
+            path = os.path.join(dirn, 'test.lmdb')
+
+            mapsize = s_const.kibibyte * 32
+            async with await s_lmdbslab.Slab.anit(path, map_size=mapsize) as slab:
+
+                self.eq(slab.mapsize, mapsize)
+
+                def slabset(key, valu):
+                    slab.replace(s_msgpack.en(key), s_msgpack.en(valu))
+
+                def slabitems():
+                    for lkey, lval in slab.scanByFull():
+                        yield s_msgpack.un(lkey), s_msgpack.un(lval)
+
+                slabset('one', [1, 2, 3])
+                slabset('two', [2, 3, 4])
+
+                seen = set()
+                for key, valu in slabitems():
+                    self.notin(key, seen)
+                    seen.add(key)
+
+                    valu += ('A' * mapsize,)
+                    slabset(key, valu)
+
+                self.gt(slab.mapsize, mapsize)
 
 class LmdbSlabMemLockTest(s_t_utils.SynTest):
 

{synapse-2.187.0 → synapse-2.187.1}/synapse/tests/test_lib_modelrev.py

@@ -1630,14 +1630,14 @@ class ModelRevTest(s_tests.SynTest):
                           sources: ['008af0047a8350287cde7abe31a7c706', 'a7a4739e0a52674df0fa3a8226de0c3f']
                           refs:
                             layer: {fork01layr}
-                              - it:prod:soft:cpe (iden: 9742664e24fe1a3a37d871b1f62af27453c2945b98f421d753db8436e9a44cc9
-                              - _ext:model:form:cpe (iden: 16e3289346a258c3e3073affad490c1d6ebf1d01295aacc489cdb24658ebc6e7
-                              - inet:flow:dst:cpes (iden: 7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7
-                              - inet:flow:src:cpes (iden: 7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7
-                              - meta:seen:node (iden: 81973208bc0f5b99250e4cda7889c66e0573c0573bc2a279083d23426ba3c74d
-                              - meta:seen:node (iden: 85bfc442d87a64a8e75d4ff2831281fb156317767612eef9b75c271ff162c4d9
+                              - it:prod:soft:cpe (iden: 9742664e24fe1a3a37d871b1f62af27453c2945b98f421d753db8436e9a44cc9)
+                              - _ext:model:form:cpe (iden: 16e3289346a258c3e3073affad490c1d6ebf1d01295aacc489cdb24658ebc6e7)
+                              - inet:flow:dst:cpes (iden: 7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7)
+                              - inet:flow:src:cpes (iden: 7d4c31f1364aaf0b4cfaf4b57bb60157f2e86248391ce8ec75d6b7e3cd5f35b7)
+                              - meta:seen:node (iden: 81973208bc0f5b99250e4cda7889c66e0573c0573bc2a279083d23426ba3c74d)
+                              - meta:seen:node (iden: 85bfc442d87a64a8e75d4ff2831281fb156317767612eef9b75c271ff162c4d9)
                             layer: {fork00layr}
-                              - risk:vulnerable:node (iden: 5fddf1b5fa06aa8a39a1eb297712cecf9ca146764c4d6e5c79296b9e9978d2c3
+                              - risk:vulnerable:node (iden: 5fddf1b5fa06aa8a39a1eb297712cecf9ca146764c4d6e5c79296b9e9978d2c3)
                           edges:
                             -(refs)> f0315900f365f45f2e027edc66ed8477d8661dad501d51f3ac8067c36565f07c
                             <(seen)- 051d93252abe655e43265b89149b6a2d5a8f5f2df33b56c986ab8671c081e394

{synapse-2.187.0 → synapse-2.187.1}/synapse/tests/test_lib_spooled.py

@@ -125,6 +125,8 @@ class SpooledTest(s_test.SynTest):
 
         async with await s_spooled.Dict.anit(size=2) as sd0:
             await runtest(sd0)
+            self.true(sd0.fallback)
 
         async with await s_spooled.Dict.anit(size=1000) as sd1:
             await runtest(sd1)
+            self.false(sd1.fallback)

{synapse-2.187.0 → synapse-2.187.1}/synapse/tests/test_utils_getrefs.py

@@ -1,10 +1,13 @@
-import os.path
+import os
 import pathlib
 
 import vcr
 
+from unittest import mock
+
 import synapse.exc as s_exc
 import synapse.data as s_data
+import synapse.common as s_common
 
 import synapse.lib.config as s_config
 
@@ -24,7 +27,7 @@ class TestUtilsGetrefs(s_utils.SynTest):
         cm = myvcr.use_cassette(fp)
         return cm
 
-    async def test_basics(self):
+    def test_basics(self):
 
         args = s_getrefs.parse_args([
             s_data.path('attack-flow', 'attack-flow-schema-2.0.0.json')
@@ -45,30 +48,34 @@ class TestUtilsGetrefs(s_utils.SynTest):
         with self.raises(s_exc.BadArg):
             s_getrefs.download_refs_handler('http://raw.githubusercontent.com/../../attack-flow/attack-flow-schema-2.0.0.json')
 
-        filename = pathlib.Path(s_data.path(
-            'jsonschemas',
-            'raw.githubusercontent.com',
-            'oasis-open',
-            'cti-stix2-json-schemas',
-            'stix2.1',
-            'schemas',
-            'common',
-            'core.json'
-        ))
-
-        self.true(filename.exists())
-        filename.unlink()
-
-        # Clear the cached validator funcs so the ref handlers in s_getrefs get called
-        s_config._JsValidators = {}
-
-        with self.getLoggerStream('synapse.utils.getrefs') as stream:
-            with self.getVcr() as cass:
-                s_getrefs.main(args)
-
-        stream.seek(0)
-        mesgs = stream.read()
-        mesg = f'Downloading schema from {CORE_URL}.'
-        self.true(filename.exists())
-        self.isin(mesg, mesgs)
-        self.notin(f'Schema {CORE_URL} already exists in local cache, skipping.', mesgs)
+        with self.getTestDir(copyfrom=s_getrefs.BASEDIR) as dirn:
+
+            filename = pathlib.Path(s_common.genpath(
+                dirn,
+                'raw.githubusercontent.com',
+                'oasis-open',
+                'cti-stix2-json-schemas',
+                'stix2.1',
+                'schemas',
+                'common',
+                'core.json'
+            ))
+
+            self.true(filename.exists())
+            filename.unlink()
+            self.false(filename.exists())
+
+            # Clear the cached validator funcs so the ref handlers in s_getrefs get called
+            s_config._JsValidators = {}
+
+            with self.getLoggerStream('synapse.utils.getrefs') as stream:
+                with mock.patch('synapse.utils.getrefs.BASEDIR', dirn):
+                    with self.getVcr():
+                        s_getrefs.main(args)
+
+            stream.seek(0)
+            mesgs = stream.read()
+            mesg = f'Downloading schema from {CORE_URL}.'
+            self.true(filename.exists())
+            self.isin(mesg, mesgs)
+            self.notin(f'Schema {CORE_URL} already exists in local cache, skipping.', mesgs)

{synapse-2.187.0 → synapse-2.187.1}/synapse/utils/getrefs.py

@@ -1,11 +1,12 @@
 import sys
 import json
 import urllib
+import asyncio
 import logging
 import pathlib
 import argparse
 
-import requests
+import aiohttp
 
 import synapse.exc as s_exc
 import synapse.data as s_data
@@ -15,23 +16,31 @@ import synapse.lib.config as s_config
 
 logger = logging.getLogger(__name__)
 
+BASEDIR = s_data.path('jsonschemas')
+
 def download_refs_handler(uri):
     '''
     This function downloads the JSON schema at the given URI, parses the given
     URI to get the path component, and then saves the referenced schema to the
     'jsonschemas' directory of synapse.data.
+
+    This function runs its own asyncio loop for each URI being requested.
     '''
+    ret = asyncio.run(_download_refs_handler(uri))
+    return ret
+
+async def _download_refs_handler(uri):
 
     try:
         parts = urllib.parse.urlparse(uri)
     except ValueError:
         raise s_exc.BadUrl(mesg=f'Malformed URI: {uri}.') from None
 
-    filename = s_data.path('jsonschemas', parts.hostname, *parts.path.split('/'))
+    filename = s_data.path(BASEDIR, parts.hostname, *parts.path.split('/'))
     filepath = pathlib.Path(filename)
 
     # Check for path traversal. Unlikely, but still check
-    if not str(filepath.absolute()).startswith(s_data.path('jsonschemas')):
+    if not str(filepath.absolute()).startswith(BASEDIR):
         raise s_exc.BadArg(mesg=f'Path traversal in schema URL: {uri} ?')
 
     # If we already have the file, return it
@@ -45,8 +54,12 @@ def download_refs_handler(uri):
 
     # Get the data from the interwebs
     logger.info(f'Downloading schema from {uri}.')
-    resp = requests.get(uri)
-    data = resp.json()
+    async with aiohttp.ClientSession() as session:
+        async with session.get(uri) as resp:
+            resp.raise_for_status()
+            buf = await resp.read()
+
+    data = json.loads(buf.decode())
 
     # Save the json schema to disk
     with filepath.open('w') as fp:

{synapse-2.187.0 → synapse-2.187.1/synapse.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: synapse
-Version: 2.187.0
+Version: 2.187.1
 Summary: Synapse Intelligence Analysis Framework
 Author-email: The Vertex Project LLC <root@vertex.link>
 License: Apache License 2.0
@@ -19,7 +19,7 @@ Classifier: Operating System :: POSIX :: Linux
 Requires-Python: >=3.11
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: pyOpenSSL<25.0.0,>=24.0.0
+Requires-Dist: pyOpenSSL<24.3.0,>=24.0.0
 Requires-Dist: cryptography<44.0.0,>=43.0.1
 Requires-Dist: msgpack<1.1.0,>=1.0.5
 Requires-Dist: xxhash<3.6.0,>=1.4.4
@@ -63,6 +63,7 @@ Requires-Dist: sphinx<9.0.0,>=8.0.0; extra == "docs"
 Requires-Dist: sphinx-rtd-theme<4.0.0,>=3.0.0; extra == "docs"
 Requires-Dist: sphinx-notfound-page<2.0.0,>=1.0.4; extra == "docs"
 Requires-Dist: jinja2<4.0.0,>=3.1.4; extra == "docs"
+Dynamic: license-file
 
 Synapse
 =======

{synapse-2.187.0 → synapse-2.187.1}/synapse.egg-info/requires.txt

@@ -1,4 +1,4 @@
-pyOpenSSL<25.0.0,>=24.0.0
+pyOpenSSL<24.3.0,>=24.0.0
 cryptography<44.0.0,>=43.0.1
 msgpack<1.1.0,>=1.0.5
 xxhash<3.6.0,>=1.4.4