synapse 2.180.0__py311-none-any.whl → 2.181.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/assets/__init__.py +35 -0
- synapse/assets/storm/migrations/model-0.2.28.storm +355 -0
- synapse/common.py +2 -1
- synapse/cortex.py +49 -35
- synapse/cryotank.py +1 -1
- synapse/datamodel.py +30 -0
- synapse/lib/ast.py +12 -7
- synapse/lib/cell.py +1 -1
- synapse/lib/certdir.py +4 -4
- synapse/lib/chop.py +0 -1
- synapse/lib/drive.py +8 -8
- synapse/lib/layer.py +55 -13
- synapse/lib/lmdbslab.py +26 -5
- synapse/lib/modelrev.py +28 -1
- synapse/lib/modules.py +1 -0
- synapse/lib/nexus.py +1 -1
- synapse/lib/node.py +5 -0
- synapse/lib/parser.py +23 -16
- synapse/lib/scrape.py +1 -1
- synapse/lib/slabseqn.py +2 -2
- synapse/lib/snap.py +129 -0
- synapse/lib/storm.lark +16 -2
- synapse/lib/storm.py +3 -0
- synapse/lib/storm_format.py +1 -0
- synapse/lib/stormhttp.py +34 -1
- synapse/lib/stormlib/auth.py +1 -1
- synapse/lib/stormlib/cortex.py +5 -2
- synapse/lib/stormlib/ipv6.py +2 -2
- synapse/lib/stormlib/model.py +114 -12
- synapse/lib/stormlib/project.py +1 -1
- synapse/lib/stormtypes.py +81 -7
- synapse/lib/types.py +7 -0
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +47 -0
- synapse/models/inet.py +10 -3
- synapse/models/infotech.py +2 -1
- synapse/models/language.py +4 -0
- synapse/models/math.py +50 -0
- synapse/models/orgs.py +8 -0
- synapse/models/risk.py +9 -0
- synapse/tests/files/stormcov/pragma-nocov.storm +18 -0
- synapse/tests/test_assets.py +25 -0
- synapse/tests/test_cortex.py +129 -0
- synapse/tests/test_datamodel.py +6 -0
- synapse/tests/test_lib_aha.py +2 -1
- synapse/tests/test_lib_certdir.py +9 -0
- synapse/tests/test_lib_grammar.py +7 -1
- synapse/tests/test_lib_layer.py +35 -0
- synapse/tests/test_lib_lmdbslab.py +11 -9
- synapse/tests/test_lib_modelrev.py +655 -1
- synapse/tests/test_lib_slabseqn.py +5 -4
- synapse/tests/test_lib_snap.py +4 -0
- synapse/tests/test_lib_storm.py +72 -1
- synapse/tests/test_lib_stormhttp.py +99 -1
- synapse/tests/test_lib_stormlib_cortex.py +21 -4
- synapse/tests/test_lib_stormlib_iters.py +8 -5
- synapse/tests/test_lib_stormlib_model.py +45 -6
- synapse/tests/test_lib_stormtypes.py +158 -2
- synapse/tests/test_lib_types.py +6 -0
- synapse/tests/test_model_inet.py +10 -0
- synapse/tests/test_model_language.py +4 -0
- synapse/tests/test_model_math.py +22 -0
- synapse/tests/test_model_orgs.py +6 -2
- synapse/tests/test_model_risk.py +4 -0
- synapse/tests/test_utils_stormcov.py +5 -0
- synapse/tests/utils.py +18 -5
- synapse/utils/stormcov/plugin.py +31 -1
- synapse/vendor/cpython/LICENSE +279 -0
- synapse/vendor/cpython/__init__.py +0 -0
- synapse/vendor/cpython/lib/__init__.py +0 -0
- synapse/vendor/cpython/lib/email/__init__.py +0 -0
- synapse/vendor/cpython/lib/email/_parseaddr.py +560 -0
- synapse/vendor/cpython/lib/email/utils.py +505 -0
- synapse/vendor/cpython/lib/ipaddress.py +2366 -0
- synapse/vendor/cpython/lib/test/__init__.py +0 -0
- synapse/vendor/cpython/lib/test/support/__init__.py +114 -0
- synapse/vendor/cpython/lib/test/test_email/__init__.py +0 -0
- synapse/vendor/cpython/lib/test/test_email/test_email.py +480 -0
- synapse/vendor/cpython/lib/test/test_email/test_utils.py +167 -0
- synapse/vendor/cpython/lib/test/test_ipaddress.py +2672 -0
- synapse/vendor/utils.py +4 -3
- {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/METADATA +2 -2
- {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/RECORD +86 -69
- {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/WHEEL +1 -1
- synapse/lib/jupyter.py +0 -505
- synapse/tests/test_lib_jupyter.py +0 -224
- {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/LICENSE +0 -0
- {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/top_level.txt +0 -0
synapse/lib/snap.py
CHANGED
|
@@ -1380,6 +1380,11 @@ class Snap(s_base.Base):
|
|
|
1380
1380
|
mesg = 'The snapshot is in read-only mode.'
|
|
1381
1381
|
raise s_exc.IsReadOnly(mesg=mesg)
|
|
1382
1382
|
|
|
1383
|
+
if isinstance(valu, dict):
|
|
1384
|
+
form = self.core.model.reqForm(name)
|
|
1385
|
+
if isinstance(form.type, s_types.Guid):
|
|
1386
|
+
return await self._addGuidNodeByDict(form, valu, props=props)
|
|
1387
|
+
|
|
1383
1388
|
async with self.getEditor() as editor:
|
|
1384
1389
|
protonode = await editor.addNode(name, valu, props=props, norminfo=norminfo)
|
|
1385
1390
|
if protonode is None:
|
|
@@ -1388,6 +1393,130 @@ class Snap(s_base.Base):
|
|
|
1388
1393
|
# the newly constructed node is cached
|
|
1389
1394
|
return await self.getNodeByBuid(protonode.buid)
|
|
1390
1395
|
|
|
1396
|
+
async def _addGuidNodeByDict(self, form, vals, props=None):
|
|
1397
|
+
|
|
1398
|
+
norms = {}
|
|
1399
|
+
counts = []
|
|
1400
|
+
proplist = []
|
|
1401
|
+
|
|
1402
|
+
if props is None:
|
|
1403
|
+
props = {}
|
|
1404
|
+
|
|
1405
|
+
trycast = vals.pop('$try', False)
|
|
1406
|
+
addprops = vals.pop('$props', None)
|
|
1407
|
+
if addprops is not None:
|
|
1408
|
+
props.update(addprops)
|
|
1409
|
+
|
|
1410
|
+
try:
|
|
1411
|
+
for name, valu in list(props.items()):
|
|
1412
|
+
props[name] = form.reqProp(name).type.norm(valu)
|
|
1413
|
+
|
|
1414
|
+
for name, valu in vals.items():
|
|
1415
|
+
|
|
1416
|
+
prop = form.reqProp(name)
|
|
1417
|
+
norm, norminfo = prop.type.norm(valu)
|
|
1418
|
+
|
|
1419
|
+
norms[name] = (prop, norm, norminfo)
|
|
1420
|
+
proplist.append((name, norm))
|
|
1421
|
+
except s_exc.BadTypeValu as e:
|
|
1422
|
+
if not trycast: raise
|
|
1423
|
+
mesg = e.errinfo.get('mesg')
|
|
1424
|
+
await self.warn(f'Bad value for prop {name}: {mesg}')
|
|
1425
|
+
return
|
|
1426
|
+
|
|
1427
|
+
proplist.sort()
|
|
1428
|
+
|
|
1429
|
+
# check first for an exact match via our same deconf strategy
|
|
1430
|
+
iden = s_common.guid(proplist)
|
|
1431
|
+
|
|
1432
|
+
node = await self.getNodeByNdef((form.full, iden))
|
|
1433
|
+
if node is not None:
|
|
1434
|
+
|
|
1435
|
+
# ensure we still match the property deconf criteria
|
|
1436
|
+
for name, (prop, norm, info) in norms.items():
|
|
1437
|
+
if not self._filtByPropAlts(node, prop, norm):
|
|
1438
|
+
break
|
|
1439
|
+
else:
|
|
1440
|
+
# ensure the non-deconf props are set
|
|
1441
|
+
async with self.getEditor() as editor:
|
|
1442
|
+
proto = editor.loadNode(node)
|
|
1443
|
+
for name, (valu, info) in props.items():
|
|
1444
|
+
await proto.set(name, valu, norminfo=info)
|
|
1445
|
+
return node
|
|
1446
|
+
|
|
1447
|
+
# TODO there is an opportunity here to populate
|
|
1448
|
+
# a look-aside for the alternative iden to speed
|
|
1449
|
+
# up future deconfliction and potentially pop them
|
|
1450
|
+
# if we lookup a node and it no longer passes the
|
|
1451
|
+
# filter...
|
|
1452
|
+
|
|
1453
|
+
# no exact match. lets do some counting.
|
|
1454
|
+
for name, (prop, norm, info) in norms.items():
|
|
1455
|
+
count = await self._getPropAltCount(prop, norm)
|
|
1456
|
+
counts.append((count, prop, norm))
|
|
1457
|
+
|
|
1458
|
+
counts.sort(key=lambda x: x[0])
|
|
1459
|
+
|
|
1460
|
+
# lift starting with the lowest count
|
|
1461
|
+
count, prop, norm = counts[0]
|
|
1462
|
+
async for node in self._nodesByPropAlts(prop, norm):
|
|
1463
|
+
await asyncio.sleep(0)
|
|
1464
|
+
|
|
1465
|
+
# filter on the remaining props/alts
|
|
1466
|
+
for count, prop, norm in counts[1:]:
|
|
1467
|
+
if not self._filtByPropAlts(node, prop, norm):
|
|
1468
|
+
break
|
|
1469
|
+
else:
|
|
1470
|
+
# ensure the non-deconf props are set
|
|
1471
|
+
async with self.getEditor() as editor:
|
|
1472
|
+
proto = editor.loadNode(node)
|
|
1473
|
+
for name, (valu, info) in props.items():
|
|
1474
|
+
await proto.set(name, valu, norminfo=info)
|
|
1475
|
+
return node
|
|
1476
|
+
|
|
1477
|
+
async with self.getEditor() as editor:
|
|
1478
|
+
proto = await editor.addNode(form.name, iden)
|
|
1479
|
+
for name, (prop, valu, info) in norms.items():
|
|
1480
|
+
await proto.set(name, valu, norminfo=info)
|
|
1481
|
+
for name, (valu, info) in props.items():
|
|
1482
|
+
await proto.set(name, valu, norminfo=info)
|
|
1483
|
+
|
|
1484
|
+
return await self.getNodeByBuid(proto.buid)
|
|
1485
|
+
|
|
1486
|
+
async def _getPropAltCount(self, prop, valu):
|
|
1487
|
+
count = 0
|
|
1488
|
+
proptype = prop.type
|
|
1489
|
+
for prop in prop.getAlts():
|
|
1490
|
+
if prop.type.isarray and prop.type.arraytype == proptype:
|
|
1491
|
+
count += await self.view.getPropArrayCount(prop.full, valu=valu)
|
|
1492
|
+
else:
|
|
1493
|
+
count += await self.view.getPropCount(prop.full, valu=valu)
|
|
1494
|
+
return count
|
|
1495
|
+
|
|
1496
|
+
def _filtByPropAlts(self, node, prop, valu):
|
|
1497
|
+
# valu must be normalized in advance
|
|
1498
|
+
proptype = prop.type
|
|
1499
|
+
for prop in prop.getAlts():
|
|
1500
|
+
if prop.type.isarray and prop.type.arraytype == proptype:
|
|
1501
|
+
if valu in node.get(prop.name):
|
|
1502
|
+
return True
|
|
1503
|
+
else:
|
|
1504
|
+
if node.get(prop.name) == valu:
|
|
1505
|
+
return True
|
|
1506
|
+
|
|
1507
|
+
return False
|
|
1508
|
+
|
|
1509
|
+
async def _nodesByPropAlts(self, prop, valu):
|
|
1510
|
+
# valu must be normalized in advance
|
|
1511
|
+
proptype = prop.type
|
|
1512
|
+
for prop in prop.getAlts():
|
|
1513
|
+
if prop.type.isarray and prop.type.arraytype == proptype:
|
|
1514
|
+
async for node in self.nodesByPropArray(prop.full, '=', valu, norm=False):
|
|
1515
|
+
yield node
|
|
1516
|
+
else:
|
|
1517
|
+
async for node in self.nodesByPropValu(prop.full, '=', valu, norm=False):
|
|
1518
|
+
yield node
|
|
1519
|
+
|
|
1391
1520
|
async def addFeedNodes(self, name, items):
|
|
1392
1521
|
'''
|
|
1393
1522
|
Call a feed function and return what it returns (typically yields Node()s).
|
synapse/lib/storm.lark
CHANGED
|
@@ -97,9 +97,23 @@ YIELD.1: /yield(?=[\s\{])/
|
|
|
97
97
|
|
|
98
98
|
// Split out case subqueries to prevent invalid state merging
|
|
99
99
|
casesubquery: "{" query "}" -> baresubquery
|
|
100
|
-
|
|
100
|
+
_caseentry: DOUBLEQUOTEDSTRING | SINGLEQUOTEDSTRING | CASEBARE
|
|
101
|
+
_mcaseentry: DOUBLEQUOTEDSTRING | SINGLEQUOTEDSTRING | MCASEBARE
|
|
102
|
+
caseentry: (DEFAULTCASE | _caseentry | "(" _mcaseentry ("," _mcaseentry)+ ")") ":" casesubquery
|
|
103
|
+
switchcase: "switch" _varvalu "{" caseentry* "}"
|
|
101
104
|
DEFAULTCASE.2: "*"
|
|
102
|
-
CASEBARE.2: /
|
|
105
|
+
CASEBARE.2: /
|
|
106
|
+
(?<=[\s{]) # can follow whitespace or the opening curly brace of the switch statement
|
|
107
|
+
(?!\*) # should not include the DEFAULTCASE bare asterisk
|
|
108
|
+
([^:\s"'()]+) # can be anything except these characters
|
|
109
|
+
(?=[\s]*:[\s]*\{) # must precede a colon and the open brace of the subquery (with any whitespace in between)
|
|
110
|
+
/x
|
|
111
|
+
MCASEBARE.2: /
|
|
112
|
+
(?<=[\s(,]) # can follow whitespace, open parens, or comma
|
|
113
|
+
([^:\s"'(),]+) # can be anything except these characters
|
|
114
|
+
(?=\s*(?:,|\)\s*:)) # must precede a comma or a close parens with colon (with any whitespace in between)
|
|
115
|
+
/x
|
|
116
|
+
|
|
103
117
|
|
|
104
118
|
yieldvalu: YIELD _argvalu
|
|
105
119
|
|
synapse/lib/storm.py
CHANGED
|
@@ -5505,6 +5505,9 @@ class ScrapeCmd(Cmd):
|
|
|
5505
5505
|
# Scrape only the :engine and :text props from the inbound nodes.
|
|
5506
5506
|
inet:search:query | scrape :text :engine
|
|
5507
5507
|
|
|
5508
|
+
# Scrape the primary property from the inbound nodes.
|
|
5509
|
+
it:dev:str | scrape $node.repr()
|
|
5510
|
+
|
|
5508
5511
|
# Scrape properties inbound nodes and yield newly scraped nodes.
|
|
5509
5512
|
inet:search:query | scrape --yield
|
|
5510
5513
|
|
synapse/lib/storm_format.py
CHANGED
synapse/lib/stormhttp.py
CHANGED
|
@@ -464,12 +464,28 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
464
464
|
kwargs['json'] = json
|
|
465
465
|
|
|
466
466
|
async with sess.request(meth, url, headers=headers, **kwargs) as resp:
|
|
467
|
+
history = []
|
|
468
|
+
for hist in resp.history:
|
|
469
|
+
hnfo = {
|
|
470
|
+
'code': hist.status,
|
|
471
|
+
'reason': await self.codereason(hist.status),
|
|
472
|
+
'headers': dict(hist.headers),
|
|
473
|
+
'url': str(hist.url),
|
|
474
|
+
# aiohttp has already closed the connection by this point
|
|
475
|
+
# so there is no connection to read a body from.
|
|
476
|
+
'body': b'',
|
|
477
|
+
'history': [],
|
|
478
|
+
'request_headers': dict(hist.request_info.headers)
|
|
479
|
+
}
|
|
480
|
+
history.append(hnfo)
|
|
467
481
|
info = {
|
|
468
482
|
'code': resp.status,
|
|
469
483
|
'reason': await self.codereason(resp.status),
|
|
470
484
|
'headers': dict(resp.headers),
|
|
471
485
|
'url': str(resp.url),
|
|
472
486
|
'body': await resp.read(),
|
|
487
|
+
'history': history,
|
|
488
|
+
'request_headers': dict(resp.request_info.headers)
|
|
473
489
|
}
|
|
474
490
|
return HttpResp(info)
|
|
475
491
|
|
|
@@ -485,12 +501,14 @@ class LibHttp(s_stormtypes.Lib):
|
|
|
485
501
|
reason = f'Exception occurred during request: {err[0]}'
|
|
486
502
|
|
|
487
503
|
info = {
|
|
504
|
+
'err': err,
|
|
488
505
|
'code': -1,
|
|
489
506
|
'reason': reason,
|
|
490
507
|
'headers': dict(),
|
|
491
508
|
'url': url,
|
|
492
509
|
'body': b'',
|
|
493
|
-
'
|
|
510
|
+
'history': [],
|
|
511
|
+
'request_headers': dict(),
|
|
494
512
|
}
|
|
495
513
|
return HttpResp(info)
|
|
496
514
|
|
|
@@ -505,7 +523,13 @@ class HttpResp(s_stormtypes.Prim):
|
|
|
505
523
|
{'name': 'reason', 'desc': 'The reason phrase for the HTTP status code.', 'type': 'str'},
|
|
506
524
|
{'name': 'body', 'desc': 'The raw HTTP response body as bytes.', 'type': 'bytes', },
|
|
507
525
|
{'name': 'headers', 'type': 'dict', 'desc': 'The HTTP Response headers.'},
|
|
526
|
+
{'name': 'request_headers', 'type': 'dict', 'desc': 'The HTTP Request headers.'},
|
|
527
|
+
{'name': 'url', 'type': 'str',
|
|
528
|
+
'desc': 'The response URL. If the request was redirected, this would be the final URL in the redirection chain. If the status code is -1, then this is the request URL.'},
|
|
508
529
|
{'name': 'err', 'type': 'list', 'desc': 'Tuple of the error type and information if an exception occurred.'},
|
|
530
|
+
{'name': 'history', 'desc': 'A list of response objects representing the history of the response. This is populated when responses are redirected.',
|
|
531
|
+
'type': {'type': 'gtor', '_gtorfunc': '_gtorHistory',
|
|
532
|
+
'returns': {'type': 'list', 'desc': 'A list of ``inet:http:resp`` objects.', }}},
|
|
509
533
|
{'name': 'json', 'desc': 'Get the JSON deserialized response.',
|
|
510
534
|
'type': {'type': 'function', '_funcname': '_httpRespJson',
|
|
511
535
|
'args': (
|
|
@@ -525,12 +549,18 @@ class HttpResp(s_stormtypes.Prim):
|
|
|
525
549
|
def __init__(self, valu, path=None):
|
|
526
550
|
super().__init__(valu, path=path)
|
|
527
551
|
self.locls.update(self.getObjLocals())
|
|
552
|
+
self.locls['url'] = self.valu.get('url')
|
|
528
553
|
self.locls['code'] = self.valu.get('code')
|
|
529
554
|
self.locls['reason'] = self.valu.get('reason')
|
|
530
555
|
self.locls['body'] = self.valu.get('body')
|
|
531
556
|
self.locls['headers'] = self.valu.get('headers')
|
|
557
|
+
self.locls['request_headers'] = self.valu.get('request_headers')
|
|
532
558
|
self.locls['err'] = self.valu.get('err', ())
|
|
533
559
|
|
|
560
|
+
self.gtors.update({
|
|
561
|
+
'history': self._gtorHistory,
|
|
562
|
+
})
|
|
563
|
+
|
|
534
564
|
def getObjLocals(self):
|
|
535
565
|
return {
|
|
536
566
|
'json': self._httpRespJson,
|
|
@@ -561,3 +591,6 @@ class HttpResp(s_stormtypes.Prim):
|
|
|
561
591
|
unpk = s_msgpack.Unpk()
|
|
562
592
|
for _, item in unpk.feed(byts):
|
|
563
593
|
yield item
|
|
594
|
+
|
|
595
|
+
async def _gtorHistory(self):
|
|
596
|
+
return [HttpResp(hnfo) for hnfo in self.valu.get('history')]
|
synapse/lib/stormlib/auth.py
CHANGED
|
@@ -877,7 +877,7 @@ class User(s_stormtypes.Prim):
|
|
|
877
877
|
''',
|
|
878
878
|
'type': {'type': 'function', '_funcname': '_methUserSetRoles',
|
|
879
879
|
'args': (
|
|
880
|
-
{'name': 'idens', 'type': 'list', 'desc': 'The idens
|
|
880
|
+
{'name': 'idens', 'type': 'list', 'desc': 'The idens of the Roles to set on the User.', },
|
|
881
881
|
),
|
|
882
882
|
'returns': {'type': 'null', }}},
|
|
883
883
|
{'name': 'revoke', 'desc': 'Remove a Role from the User',
|
synapse/lib/stormlib/cortex.py
CHANGED
|
@@ -1085,7 +1085,7 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1085
1085
|
{'name': 'path', 'type': 'string',
|
|
1086
1086
|
'desc': 'The extended HTTP API path.'},
|
|
1087
1087
|
{'name': 'name', 'type': 'string',
|
|
1088
|
-
'desc': 'Friendly name for the Extended HTTP API', 'default': ''},
|
|
1088
|
+
'desc': 'Friendly name for the Extended HTTP API.', 'default': ''},
|
|
1089
1089
|
{'name': 'desc', 'type': 'string',
|
|
1090
1090
|
'desc': 'Description for the Extended HTTP API.', 'default': ''},
|
|
1091
1091
|
{'name': 'runas', 'type': 'string',
|
|
@@ -1095,6 +1095,8 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1095
1095
|
'desc': 'Require the API endpoint to be authenticated.', 'default': True},
|
|
1096
1096
|
{'name': 'readonly', 'type': 'boolean',
|
|
1097
1097
|
'desc': 'Run the Extended HTTP Storm methods in readonly mode.', 'default': False},
|
|
1098
|
+
{'name': 'iden', 'type': 'str',
|
|
1099
|
+
'desc': 'An iden for the new Extended HTTP API.', 'default': None},
|
|
1098
1100
|
),
|
|
1099
1101
|
'returns': {'type': 'http:api', 'desc': 'A new ``http:api`` object.'}}},
|
|
1100
1102
|
{'name': 'del', 'desc': 'Delete an Extended HTTP API endpoint.',
|
|
@@ -1196,7 +1198,7 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1196
1198
|
apis = [HttpApi(self.runt, adef) for adef in adefs]
|
|
1197
1199
|
return apis
|
|
1198
1200
|
|
|
1199
|
-
async def addHttpApi(self, path, name='', desc='', runas='owner', authenticated=True, readonly=False):
|
|
1201
|
+
async def addHttpApi(self, path, name='', desc='', runas='owner', authenticated=True, readonly=False, iden=None):
|
|
1200
1202
|
s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'add'))
|
|
1201
1203
|
|
|
1202
1204
|
path = await s_stormtypes.tostr(path)
|
|
@@ -1207,6 +1209,7 @@ class CortexHttpApi(s_stormtypes.Lib):
|
|
|
1207
1209
|
authenticated = await s_stormtypes.tobool(authenticated)
|
|
1208
1210
|
|
|
1209
1211
|
adef = {
|
|
1212
|
+
'iden': iden,
|
|
1210
1213
|
'path': path,
|
|
1211
1214
|
'view': self.runt.snap.view.iden,
|
|
1212
1215
|
'runas': runas,
|
synapse/lib/stormlib/ipv6.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import logging
|
|
2
|
-
import ipaddress
|
|
3
|
-
|
|
4
2
|
|
|
5
3
|
import synapse.exc as s_exc
|
|
4
|
+
import synapse.common as s_common
|
|
6
5
|
|
|
7
6
|
import synapse.lib.stormtypes as s_stormtypes
|
|
8
7
|
|
|
8
|
+
ipaddress = s_common.ipaddress
|
|
9
9
|
logger = logging.getLogger(__name__)
|
|
10
10
|
|
|
11
11
|
@s_stormtypes.registry.registerLib
|
synapse/lib/stormlib/model.py
CHANGED
|
@@ -3,6 +3,7 @@ import synapse.common as s_common
|
|
|
3
3
|
|
|
4
4
|
import synapse.lib.node as s_node
|
|
5
5
|
import synapse.lib.cache as s_cache
|
|
6
|
+
import synapse.lib.layer as s_layer
|
|
6
7
|
import synapse.lib.stormtypes as s_stormtypes
|
|
7
8
|
|
|
8
9
|
import synapse.models.infotech as s_infotech
|
|
@@ -497,6 +498,7 @@ class ModelType(s_stormtypes.Prim):
|
|
|
497
498
|
_storm_locals = (
|
|
498
499
|
{'name': 'name', 'desc': 'The name of the Type.', 'type': 'str', },
|
|
499
500
|
{'name': 'stortype', 'desc': 'The storetype of the Type.', 'type': 'int', },
|
|
501
|
+
{'name': 'opts', 'desc': 'The options for the Type.', 'type': 'dict', },
|
|
500
502
|
{'name': 'repr', 'desc': 'Get the repr of a value for the Type.',
|
|
501
503
|
'type': {'type': 'function', '_funcname': '_methRepr',
|
|
502
504
|
'args': (
|
|
@@ -516,6 +518,7 @@ class ModelType(s_stormtypes.Prim):
|
|
|
516
518
|
s_stormtypes.Prim.__init__(self, valu, path=path)
|
|
517
519
|
self.locls.update(self.getObjLocals())
|
|
518
520
|
self.locls.update({'name': valu.name,
|
|
521
|
+
'opts': valu.opts,
|
|
519
522
|
'stortype': valu.stortype,
|
|
520
523
|
})
|
|
521
524
|
|
|
@@ -756,6 +759,16 @@ class MigrationEditorMixin:
|
|
|
756
759
|
if overwrite or not proto.hasTagProp(tagname, propname):
|
|
757
760
|
await proto.setTagProp(tagname, propname, valu) # use tag perms
|
|
758
761
|
|
|
762
|
+
async def copyExtProps(self, src, proto):
|
|
763
|
+
|
|
764
|
+
form = src.form
|
|
765
|
+
|
|
766
|
+
for name, valu in src.props.items():
|
|
767
|
+
prop = form.props.get(name)
|
|
768
|
+
if not prop.isext:
|
|
769
|
+
continue
|
|
770
|
+
|
|
771
|
+
await proto.set(name, valu)
|
|
759
772
|
|
|
760
773
|
@s_stormtypes.registry.registerLib
|
|
761
774
|
class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
|
|
@@ -788,6 +801,13 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
788
801
|
'desc': 'Copy tag property value even if the property exists on the destination node.', },
|
|
789
802
|
),
|
|
790
803
|
'returns': {'type': 'null', }}},
|
|
804
|
+
{'name': 'copyExtProps', 'desc': 'Copy extended properties from the src node to the dst node.',
|
|
805
|
+
'type': {'type': 'function', '_funcname': '_methCopyExtProps',
|
|
806
|
+
'args': (
|
|
807
|
+
{'name': 'src', 'type': 'node', 'desc': 'The node to copy extended props from.', },
|
|
808
|
+
{'name': 'dst', 'type': 'node', 'desc': 'The node to copy extended props to.', },
|
|
809
|
+
),
|
|
810
|
+
'returns': {'type': 'null', }}},
|
|
791
811
|
)
|
|
792
812
|
_storm_lib_path = ('model', 'migration')
|
|
793
813
|
|
|
@@ -796,6 +816,9 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
796
816
|
'copyData': self._methCopyData,
|
|
797
817
|
'copyEdges': self._methCopyEdges,
|
|
798
818
|
'copyTags': self._methCopyTags,
|
|
819
|
+
'copyExtProps': self._methCopyExtProps,
|
|
820
|
+
'liftByPropValuNoNorm': self._methLiftByPropValuNoNorm,
|
|
821
|
+
'setNodePropValuNoNorm': self._methSetNodePropValuNoNorm,
|
|
799
822
|
}
|
|
800
823
|
|
|
801
824
|
async def _methCopyData(self, src, dst, overwrite=False):
|
|
@@ -839,6 +862,79 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
839
862
|
proto = editor.loadNode(dst)
|
|
840
863
|
await self.copyTags(src, proto, overwrite=overwrite)
|
|
841
864
|
|
|
865
|
+
async def _methCopyExtProps(self, src, dst):
|
|
866
|
+
|
|
867
|
+
if not isinstance(src, s_node.Node):
|
|
868
|
+
raise s_exc.BadArg(mesg='$lib.model.migration.copyExtProps() source argument must be a node.')
|
|
869
|
+
if not isinstance(dst, s_node.Node):
|
|
870
|
+
raise s_exc.BadArg(mesg='$lib.model.migration.copyExtProps() dest argument must be a node.')
|
|
871
|
+
|
|
872
|
+
snap = self.runt.snap
|
|
873
|
+
|
|
874
|
+
async with snap.getEditor() as editor:
|
|
875
|
+
proto = editor.loadNode(dst)
|
|
876
|
+
await self.copyExtProps(src, proto)
|
|
877
|
+
|
|
878
|
+
async def _methLiftByPropValuNoNorm(self, formname, propname, valu, cmpr='=', reverse=False):
|
|
879
|
+
'''
|
|
880
|
+
No storm docs for this on purpose. It is restricted for use during model migrations only.
|
|
881
|
+
'''
|
|
882
|
+
formname = await s_stormtypes.tostr(formname)
|
|
883
|
+
propname = await s_stormtypes.tostr(propname)
|
|
884
|
+
valu = await s_stormtypes.toprim(valu)
|
|
885
|
+
|
|
886
|
+
prop = self.runt.snap.core.model.prop(f'{formname}:{propname}')
|
|
887
|
+
if prop is None:
|
|
888
|
+
mesg = f'Could not find prop: {formname}:{propname}'
|
|
889
|
+
raise s_exc.NoSuchProp(mesg=mesg, formname=formname, propname=propname)
|
|
890
|
+
|
|
891
|
+
if not self.runt.snap.core.migration:
|
|
892
|
+
mesg = '$lib.model.migration.liftByPropValuNoNorm() is restricted to model migrations only.'
|
|
893
|
+
raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
|
|
894
|
+
|
|
895
|
+
stortype = prop.type.stortype
|
|
896
|
+
|
|
897
|
+
# Normally we'd call proptype.getStorCmprs() here to get the cmprvals
|
|
898
|
+
# but getStorCmprs() calls norm() which we're trying to avoid so build
|
|
899
|
+
# cmprvals manually here.
|
|
900
|
+
|
|
901
|
+
if prop.type.isarray:
|
|
902
|
+
stortype &= (~s_layer.STOR_FLAG_ARRAY)
|
|
903
|
+
liftfunc = self.runt.snap.wlyr.liftByPropArray
|
|
904
|
+
else:
|
|
905
|
+
liftfunc = self.runt.snap.wlyr.liftByPropValu
|
|
906
|
+
|
|
907
|
+
cmprvals = ((cmpr, valu, stortype),)
|
|
908
|
+
|
|
909
|
+
layriden = self.runt.snap.wlyr.iden
|
|
910
|
+
async for _, buid, sode in liftfunc(formname, propname, cmprvals, reverse=reverse):
|
|
911
|
+
yield await self.runt.snap._joinStorNode(buid, {layriden: sode})
|
|
912
|
+
|
|
913
|
+
async def _methSetNodePropValuNoNorm(self, n, propname, valu):
|
|
914
|
+
'''
|
|
915
|
+
No storm docs for this on purpose. It is restricted for use during model migrations only.
|
|
916
|
+
'''
|
|
917
|
+
|
|
918
|
+
# NB: I'm sure there are all kinds of edges cases that this function doesn't account for. At the time of it's
|
|
919
|
+
# creation, this was intended to be used to update array properties with bad it:sec:cpe values in them. It works
|
|
920
|
+
# for that use case (see model migration 0.2.28). Any additional use of this function should perform heavy
|
|
921
|
+
# testing.
|
|
922
|
+
|
|
923
|
+
if not isinstance(n, s_node.Node):
|
|
924
|
+
raise s_exc.BadArg(mesg='$lib.model.migration.setNodePropValuNoNorm() argument must be a node.')
|
|
925
|
+
|
|
926
|
+
if not self.runt.snap.core.migration:
|
|
927
|
+
mesg = '$lib.model.migration.setNodePropValuNoNorm() is restricted to model migrations only.'
|
|
928
|
+
raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
|
|
929
|
+
|
|
930
|
+
propname = await s_stormtypes.tostr(propname)
|
|
931
|
+
valu = await s_stormtypes.toprim(valu)
|
|
932
|
+
|
|
933
|
+
async with self.runt.snap.getNodeEditor(n) as proto:
|
|
934
|
+
await proto.set(propname, valu, norminfo={})
|
|
935
|
+
|
|
936
|
+
return n
|
|
937
|
+
|
|
842
938
|
@s_stormtypes.registry.registerLib
|
|
843
939
|
class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
844
940
|
'''
|
|
@@ -980,10 +1076,15 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
980
1076
|
def getObjLocals(self):
|
|
981
1077
|
return {
|
|
982
1078
|
'itSecCpe_2_170_0': self._itSecCpe_2_170_0,
|
|
1079
|
+
'itSecCpe_2_170_0_internal': self._itSecCpe_2_170_0_internal,
|
|
983
1080
|
'riskHasVulnToVulnerable': self._riskHasVulnToVulnerable,
|
|
984
1081
|
}
|
|
985
1082
|
|
|
986
1083
|
async def _itSecCpe_2_170_0(self, n, prefer_v22=False, force=False):
|
|
1084
|
+
info = await self._itSecCpe_2_170_0_internal(n, prefer_v22=prefer_v22, force=force, set_nodedata=True)
|
|
1085
|
+
return info.get('status') == 'success'
|
|
1086
|
+
|
|
1087
|
+
async def _itSecCpe_2_170_0_internal(self, n, prefer_v22=False, force=False, set_nodedata=False):
|
|
987
1088
|
|
|
988
1089
|
if not isinstance(n, s_node.Node):
|
|
989
1090
|
raise s_exc.BadArg(mesg='$lib.model.migration.s.itSecCpe_2_170_0() argument must be a node.')
|
|
@@ -1008,13 +1109,12 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
1008
1109
|
if self.runt.debug:
|
|
1009
1110
|
mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node already migrated.'
|
|
1010
1111
|
await self.runt.printf(mesg)
|
|
1011
|
-
return
|
|
1112
|
+
return nodedata
|
|
1012
1113
|
|
|
1013
1114
|
modl = self.runt.model.type('it:sec:cpe')
|
|
1014
1115
|
|
|
1015
1116
|
valu23 = None
|
|
1016
1117
|
valu22 = None
|
|
1017
|
-
invalid = ''
|
|
1018
1118
|
|
|
1019
1119
|
# Check the primary property for validity.
|
|
1020
1120
|
cpe23 = s_infotech.cpe23_regex.match(curv)
|
|
@@ -1036,11 +1136,11 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
1036
1136
|
mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node is valid, no migration necessary.'
|
|
1037
1137
|
await self.runt.printf(mesg)
|
|
1038
1138
|
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1139
|
+
nodedata = {'status': 'success'}
|
|
1140
|
+
if set_nodedata:
|
|
1141
|
+
await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
|
|
1042
1142
|
|
|
1043
|
-
return
|
|
1143
|
+
return nodedata
|
|
1044
1144
|
|
|
1045
1145
|
if valu23 is None and valu22 is None:
|
|
1046
1146
|
reason = 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.'
|
|
@@ -1049,12 +1149,14 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
1049
1149
|
mesg = f'itSecCpe_2_170_0({reprvalu}): {reason}'
|
|
1050
1150
|
await self.runt.warn(mesg)
|
|
1051
1151
|
|
|
1052
|
-
|
|
1152
|
+
nodedata = {
|
|
1053
1153
|
'status': 'failed',
|
|
1054
1154
|
'reason': reason,
|
|
1055
|
-
}
|
|
1155
|
+
}
|
|
1156
|
+
if set_nodedata:
|
|
1157
|
+
await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
|
|
1056
1158
|
|
|
1057
|
-
return
|
|
1159
|
+
return nodedata
|
|
1058
1160
|
|
|
1059
1161
|
if prefer_v22:
|
|
1060
1162
|
valu = valu22 or valu23
|
|
@@ -1065,7 +1167,6 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
1065
1167
|
norm, info = modl.norm(valu)
|
|
1066
1168
|
subs = info.get('subs')
|
|
1067
1169
|
|
|
1068
|
-
edits = []
|
|
1069
1170
|
nodedata = {'status': 'success'}
|
|
1070
1171
|
|
|
1071
1172
|
if norm != curv:
|
|
@@ -1097,7 +1198,8 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
1097
1198
|
# Update the existing property with the re-normalized property value.
|
|
1098
1199
|
await proto.set(propname, subscurv, ignore_ro=True)
|
|
1099
1200
|
|
|
1100
|
-
|
|
1201
|
+
if set_nodedata:
|
|
1202
|
+
await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
|
|
1101
1203
|
|
|
1102
1204
|
if self.runt.debug:
|
|
1103
1205
|
if nodedata.get('updated'):
|
|
@@ -1107,7 +1209,7 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
1107
1209
|
mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): No property updates required.'
|
|
1108
1210
|
await self.runt.printf(mesg)
|
|
1109
1211
|
|
|
1110
|
-
return
|
|
1212
|
+
return nodedata
|
|
1111
1213
|
|
|
1112
1214
|
async def _riskHasVulnToVulnerable(self, n, nodata=False):
|
|
1113
1215
|
|
synapse/lib/stormlib/project.py
CHANGED
|
@@ -847,7 +847,7 @@ class LibProjects(s_stormtypes.Lib):
|
|
|
847
847
|
{'name': 'name', 'type': 'str', 'desc': 'The name of the Project to get'},
|
|
848
848
|
),
|
|
849
849
|
'returns': {'type': 'proj:project',
|
|
850
|
-
'desc': 'The
|
|
850
|
+
'desc': 'The project object, if it exists, otherwise null'}}},
|
|
851
851
|
|
|
852
852
|
{'name': 'add', 'desc': 'Add a new project',
|
|
853
853
|
'type': {'type': 'function', '_funcname': '_funcProjAdd',
|