synapse 2.180.0__py311-none-any.whl → 2.181.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (88) hide show
  1. synapse/assets/__init__.py +35 -0
  2. synapse/assets/storm/migrations/model-0.2.28.storm +355 -0
  3. synapse/common.py +2 -1
  4. synapse/cortex.py +49 -35
  5. synapse/cryotank.py +1 -1
  6. synapse/datamodel.py +30 -0
  7. synapse/lib/ast.py +12 -7
  8. synapse/lib/cell.py +1 -1
  9. synapse/lib/certdir.py +4 -4
  10. synapse/lib/chop.py +0 -1
  11. synapse/lib/drive.py +8 -8
  12. synapse/lib/layer.py +55 -13
  13. synapse/lib/lmdbslab.py +26 -5
  14. synapse/lib/modelrev.py +28 -1
  15. synapse/lib/modules.py +1 -0
  16. synapse/lib/nexus.py +1 -1
  17. synapse/lib/node.py +5 -0
  18. synapse/lib/parser.py +23 -16
  19. synapse/lib/scrape.py +1 -1
  20. synapse/lib/slabseqn.py +2 -2
  21. synapse/lib/snap.py +129 -0
  22. synapse/lib/storm.lark +16 -2
  23. synapse/lib/storm.py +3 -0
  24. synapse/lib/storm_format.py +1 -0
  25. synapse/lib/stormhttp.py +34 -1
  26. synapse/lib/stormlib/auth.py +1 -1
  27. synapse/lib/stormlib/cortex.py +5 -2
  28. synapse/lib/stormlib/ipv6.py +2 -2
  29. synapse/lib/stormlib/model.py +114 -12
  30. synapse/lib/stormlib/project.py +1 -1
  31. synapse/lib/stormtypes.py +81 -7
  32. synapse/lib/types.py +7 -0
  33. synapse/lib/version.py +2 -2
  34. synapse/lib/view.py +47 -0
  35. synapse/models/inet.py +10 -3
  36. synapse/models/infotech.py +2 -1
  37. synapse/models/language.py +4 -0
  38. synapse/models/math.py +50 -0
  39. synapse/models/orgs.py +8 -0
  40. synapse/models/risk.py +9 -0
  41. synapse/tests/files/stormcov/pragma-nocov.storm +18 -0
  42. synapse/tests/test_assets.py +25 -0
  43. synapse/tests/test_cortex.py +129 -0
  44. synapse/tests/test_datamodel.py +6 -0
  45. synapse/tests/test_lib_aha.py +2 -1
  46. synapse/tests/test_lib_certdir.py +9 -0
  47. synapse/tests/test_lib_grammar.py +7 -1
  48. synapse/tests/test_lib_layer.py +35 -0
  49. synapse/tests/test_lib_lmdbslab.py +11 -9
  50. synapse/tests/test_lib_modelrev.py +655 -1
  51. synapse/tests/test_lib_slabseqn.py +5 -4
  52. synapse/tests/test_lib_snap.py +4 -0
  53. synapse/tests/test_lib_storm.py +72 -1
  54. synapse/tests/test_lib_stormhttp.py +99 -1
  55. synapse/tests/test_lib_stormlib_cortex.py +21 -4
  56. synapse/tests/test_lib_stormlib_iters.py +8 -5
  57. synapse/tests/test_lib_stormlib_model.py +45 -6
  58. synapse/tests/test_lib_stormtypes.py +158 -2
  59. synapse/tests/test_lib_types.py +6 -0
  60. synapse/tests/test_model_inet.py +10 -0
  61. synapse/tests/test_model_language.py +4 -0
  62. synapse/tests/test_model_math.py +22 -0
  63. synapse/tests/test_model_orgs.py +6 -2
  64. synapse/tests/test_model_risk.py +4 -0
  65. synapse/tests/test_utils_stormcov.py +5 -0
  66. synapse/tests/utils.py +18 -5
  67. synapse/utils/stormcov/plugin.py +31 -1
  68. synapse/vendor/cpython/LICENSE +279 -0
  69. synapse/vendor/cpython/__init__.py +0 -0
  70. synapse/vendor/cpython/lib/__init__.py +0 -0
  71. synapse/vendor/cpython/lib/email/__init__.py +0 -0
  72. synapse/vendor/cpython/lib/email/_parseaddr.py +560 -0
  73. synapse/vendor/cpython/lib/email/utils.py +505 -0
  74. synapse/vendor/cpython/lib/ipaddress.py +2366 -0
  75. synapse/vendor/cpython/lib/test/__init__.py +0 -0
  76. synapse/vendor/cpython/lib/test/support/__init__.py +114 -0
  77. synapse/vendor/cpython/lib/test/test_email/__init__.py +0 -0
  78. synapse/vendor/cpython/lib/test/test_email/test_email.py +480 -0
  79. synapse/vendor/cpython/lib/test/test_email/test_utils.py +167 -0
  80. synapse/vendor/cpython/lib/test/test_ipaddress.py +2672 -0
  81. synapse/vendor/utils.py +4 -3
  82. {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/METADATA +2 -2
  83. {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/RECORD +86 -69
  84. {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/WHEEL +1 -1
  85. synapse/lib/jupyter.py +0 -505
  86. synapse/tests/test_lib_jupyter.py +0 -224
  87. {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/LICENSE +0 -0
  88. {synapse-2.180.0.dist-info → synapse-2.181.0.dist-info}/top_level.txt +0 -0
synapse/lib/snap.py CHANGED
@@ -1380,6 +1380,11 @@ class Snap(s_base.Base):
1380
1380
  mesg = 'The snapshot is in read-only mode.'
1381
1381
  raise s_exc.IsReadOnly(mesg=mesg)
1382
1382
 
1383
+ if isinstance(valu, dict):
1384
+ form = self.core.model.reqForm(name)
1385
+ if isinstance(form.type, s_types.Guid):
1386
+ return await self._addGuidNodeByDict(form, valu, props=props)
1387
+
1383
1388
  async with self.getEditor() as editor:
1384
1389
  protonode = await editor.addNode(name, valu, props=props, norminfo=norminfo)
1385
1390
  if protonode is None:
@@ -1388,6 +1393,130 @@ class Snap(s_base.Base):
1388
1393
  # the newly constructed node is cached
1389
1394
  return await self.getNodeByBuid(protonode.buid)
1390
1395
 
1396
+ async def _addGuidNodeByDict(self, form, vals, props=None):
1397
+
1398
+ norms = {}
1399
+ counts = []
1400
+ proplist = []
1401
+
1402
+ if props is None:
1403
+ props = {}
1404
+
1405
+ trycast = vals.pop('$try', False)
1406
+ addprops = vals.pop('$props', None)
1407
+ if addprops is not None:
1408
+ props.update(addprops)
1409
+
1410
+ try:
1411
+ for name, valu in list(props.items()):
1412
+ props[name] = form.reqProp(name).type.norm(valu)
1413
+
1414
+ for name, valu in vals.items():
1415
+
1416
+ prop = form.reqProp(name)
1417
+ norm, norminfo = prop.type.norm(valu)
1418
+
1419
+ norms[name] = (prop, norm, norminfo)
1420
+ proplist.append((name, norm))
1421
+ except s_exc.BadTypeValu as e:
1422
+ if not trycast: raise
1423
+ mesg = e.errinfo.get('mesg')
1424
+ await self.warn(f'Bad value for prop {name}: {mesg}')
1425
+ return
1426
+
1427
+ proplist.sort()
1428
+
1429
+ # check first for an exact match via our same deconf strategy
1430
+ iden = s_common.guid(proplist)
1431
+
1432
+ node = await self.getNodeByNdef((form.full, iden))
1433
+ if node is not None:
1434
+
1435
+ # ensure we still match the property deconf criteria
1436
+ for name, (prop, norm, info) in norms.items():
1437
+ if not self._filtByPropAlts(node, prop, norm):
1438
+ break
1439
+ else:
1440
+ # ensure the non-deconf props are set
1441
+ async with self.getEditor() as editor:
1442
+ proto = editor.loadNode(node)
1443
+ for name, (valu, info) in props.items():
1444
+ await proto.set(name, valu, norminfo=info)
1445
+ return node
1446
+
1447
+ # TODO there is an opportunity here to populate
1448
+ # a look-aside for the alternative iden to speed
1449
+ # up future deconfliction and potentially pop them
1450
+ # if we lookup a node and it no longer passes the
1451
+ # filter...
1452
+
1453
+ # no exact match. lets do some counting.
1454
+ for name, (prop, norm, info) in norms.items():
1455
+ count = await self._getPropAltCount(prop, norm)
1456
+ counts.append((count, prop, norm))
1457
+
1458
+ counts.sort(key=lambda x: x[0])
1459
+
1460
+ # lift starting with the lowest count
1461
+ count, prop, norm = counts[0]
1462
+ async for node in self._nodesByPropAlts(prop, norm):
1463
+ await asyncio.sleep(0)
1464
+
1465
+ # filter on the remaining props/alts
1466
+ for count, prop, norm in counts[1:]:
1467
+ if not self._filtByPropAlts(node, prop, norm):
1468
+ break
1469
+ else:
1470
+ # ensure the non-deconf props are set
1471
+ async with self.getEditor() as editor:
1472
+ proto = editor.loadNode(node)
1473
+ for name, (valu, info) in props.items():
1474
+ await proto.set(name, valu, norminfo=info)
1475
+ return node
1476
+
1477
+ async with self.getEditor() as editor:
1478
+ proto = await editor.addNode(form.name, iden)
1479
+ for name, (prop, valu, info) in norms.items():
1480
+ await proto.set(name, valu, norminfo=info)
1481
+ for name, (valu, info) in props.items():
1482
+ await proto.set(name, valu, norminfo=info)
1483
+
1484
+ return await self.getNodeByBuid(proto.buid)
1485
+
1486
+ async def _getPropAltCount(self, prop, valu):
1487
+ count = 0
1488
+ proptype = prop.type
1489
+ for prop in prop.getAlts():
1490
+ if prop.type.isarray and prop.type.arraytype == proptype:
1491
+ count += await self.view.getPropArrayCount(prop.full, valu=valu)
1492
+ else:
1493
+ count += await self.view.getPropCount(prop.full, valu=valu)
1494
+ return count
1495
+
1496
+ def _filtByPropAlts(self, node, prop, valu):
1497
+ # valu must be normalized in advance
1498
+ proptype = prop.type
1499
+ for prop in prop.getAlts():
1500
+ if prop.type.isarray and prop.type.arraytype == proptype:
1501
+ if valu in node.get(prop.name):
1502
+ return True
1503
+ else:
1504
+ if node.get(prop.name) == valu:
1505
+ return True
1506
+
1507
+ return False
1508
+
1509
+ async def _nodesByPropAlts(self, prop, valu):
1510
+ # valu must be normalized in advance
1511
+ proptype = prop.type
1512
+ for prop in prop.getAlts():
1513
+ if prop.type.isarray and prop.type.arraytype == proptype:
1514
+ async for node in self.nodesByPropArray(prop.full, '=', valu, norm=False):
1515
+ yield node
1516
+ else:
1517
+ async for node in self.nodesByPropValu(prop.full, '=', valu, norm=False):
1518
+ yield node
1519
+
1391
1520
  async def addFeedNodes(self, name, items):
1392
1521
  '''
1393
1522
  Call a feed function and return what it returns (typically yields Node()s).
synapse/lib/storm.lark CHANGED
@@ -97,9 +97,23 @@ YIELD.1: /yield(?=[\s\{])/
97
97
 
98
98
  // Split out case subqueries to prevent invalid state merging
99
99
  casesubquery: "{" query "}" -> baresubquery
100
- switchcase: "switch" _varvalu "{" ((DEFAULTCASE | DOUBLEQUOTEDSTRING | SINGLEQUOTEDSTRING | formatstring | CASEBARE) ":" casesubquery)* "}"
100
+ _caseentry: DOUBLEQUOTEDSTRING | SINGLEQUOTEDSTRING | CASEBARE
101
+ _mcaseentry: DOUBLEQUOTEDSTRING | SINGLEQUOTEDSTRING | MCASEBARE
102
+ caseentry: (DEFAULTCASE | _caseentry | "(" _mcaseentry ("," _mcaseentry)+ ")") ":" casesubquery
103
+ switchcase: "switch" _varvalu "{" caseentry* "}"
101
104
  DEFAULTCASE.2: "*"
102
- CASEBARE.2: /(?<=[\s{])(?!\*)([^:\s"']+)(?=[\s]*:[\s]*\{)/
105
+ CASEBARE.2: /
106
+ (?<=[\s{]) # can follow whitespace or the opening curly brace of the switch statement
107
+ (?!\*) # should not include the DEFAULTCASE bare asterisk
108
+ ([^:\s"'()]+) # can be anything except these characters
109
+ (?=[\s]*:[\s]*\{) # must precede a colon and the open brace of the subquery (with any whitespace in between)
110
+ /x
111
+ MCASEBARE.2: /
112
+ (?<=[\s(,]) # can follow whitespace, open parens, or comma
113
+ ([^:\s"'(),]+) # can be anything except these characters
114
+ (?=\s*(?:,|\)\s*:)) # must precede a comma or a close parens with colon (with any whitespace in between)
115
+ /x
116
+
103
117
 
104
118
  yieldvalu: YIELD _argvalu
105
119
 
synapse/lib/storm.py CHANGED
@@ -5505,6 +5505,9 @@ class ScrapeCmd(Cmd):
5505
5505
  # Scrape only the :engine and :text props from the inbound nodes.
5506
5506
  inet:search:query | scrape :text :engine
5507
5507
 
5508
+ # Scrape the primary property from the inbound nodes.
5509
+ it:dev:str | scrape $node.repr()
5510
+
5508
5511
  # Scrape properties inbound nodes and yield newly scraped nodes.
5509
5512
  inet:search:query | scrape --yield
5510
5513
 
@@ -52,6 +52,7 @@ TerminalPygMap = {
52
52
  'LISTTOKN': p_t.Literal.String,
53
53
  'LPAR': p_t.Punctuation,
54
54
  'LSQB': p_t.Punctuation,
55
+ 'MCASEBARE': p_t.Literal.String,
55
56
  'MODSET': p_t.Operator,
56
57
  'NONQUOTEWORD': p_t.Literal,
57
58
  'NOT': p_t.Keyword,
synapse/lib/stormhttp.py CHANGED
@@ -464,12 +464,28 @@ class LibHttp(s_stormtypes.Lib):
464
464
  kwargs['json'] = json
465
465
 
466
466
  async with sess.request(meth, url, headers=headers, **kwargs) as resp:
467
+ history = []
468
+ for hist in resp.history:
469
+ hnfo = {
470
+ 'code': hist.status,
471
+ 'reason': await self.codereason(hist.status),
472
+ 'headers': dict(hist.headers),
473
+ 'url': str(hist.url),
474
+ # aiohttp has already closed the connection by this point
475
+ # so there is no connection to read a body from.
476
+ 'body': b'',
477
+ 'history': [],
478
+ 'request_headers': dict(hist.request_info.headers)
479
+ }
480
+ history.append(hnfo)
467
481
  info = {
468
482
  'code': resp.status,
469
483
  'reason': await self.codereason(resp.status),
470
484
  'headers': dict(resp.headers),
471
485
  'url': str(resp.url),
472
486
  'body': await resp.read(),
487
+ 'history': history,
488
+ 'request_headers': dict(resp.request_info.headers)
473
489
  }
474
490
  return HttpResp(info)
475
491
 
@@ -485,12 +501,14 @@ class LibHttp(s_stormtypes.Lib):
485
501
  reason = f'Exception occurred during request: {err[0]}'
486
502
 
487
503
  info = {
504
+ 'err': err,
488
505
  'code': -1,
489
506
  'reason': reason,
490
507
  'headers': dict(),
491
508
  'url': url,
492
509
  'body': b'',
493
- 'err': err,
510
+ 'history': [],
511
+ 'request_headers': dict(),
494
512
  }
495
513
  return HttpResp(info)
496
514
 
@@ -505,7 +523,13 @@ class HttpResp(s_stormtypes.Prim):
505
523
  {'name': 'reason', 'desc': 'The reason phrase for the HTTP status code.', 'type': 'str'},
506
524
  {'name': 'body', 'desc': 'The raw HTTP response body as bytes.', 'type': 'bytes', },
507
525
  {'name': 'headers', 'type': 'dict', 'desc': 'The HTTP Response headers.'},
526
+ {'name': 'request_headers', 'type': 'dict', 'desc': 'The HTTP Request headers.'},
527
+ {'name': 'url', 'type': 'str',
528
+ 'desc': 'The response URL. If the request was redirected, this would be the final URL in the redirection chain. If the status code is -1, then this is the request URL.'},
508
529
  {'name': 'err', 'type': 'list', 'desc': 'Tuple of the error type and information if an exception occurred.'},
530
+ {'name': 'history', 'desc': 'A list of response objects representing the history of the response. This is populated when responses are redirected.',
531
+ 'type': {'type': 'gtor', '_gtorfunc': '_gtorHistory',
532
+ 'returns': {'type': 'list', 'desc': 'A list of ``inet:http:resp`` objects.', }}},
509
533
  {'name': 'json', 'desc': 'Get the JSON deserialized response.',
510
534
  'type': {'type': 'function', '_funcname': '_httpRespJson',
511
535
  'args': (
@@ -525,12 +549,18 @@ class HttpResp(s_stormtypes.Prim):
525
549
  def __init__(self, valu, path=None):
526
550
  super().__init__(valu, path=path)
527
551
  self.locls.update(self.getObjLocals())
552
+ self.locls['url'] = self.valu.get('url')
528
553
  self.locls['code'] = self.valu.get('code')
529
554
  self.locls['reason'] = self.valu.get('reason')
530
555
  self.locls['body'] = self.valu.get('body')
531
556
  self.locls['headers'] = self.valu.get('headers')
557
+ self.locls['request_headers'] = self.valu.get('request_headers')
532
558
  self.locls['err'] = self.valu.get('err', ())
533
559
 
560
+ self.gtors.update({
561
+ 'history': self._gtorHistory,
562
+ })
563
+
534
564
  def getObjLocals(self):
535
565
  return {
536
566
  'json': self._httpRespJson,
@@ -561,3 +591,6 @@ class HttpResp(s_stormtypes.Prim):
561
591
  unpk = s_msgpack.Unpk()
562
592
  for _, item in unpk.feed(byts):
563
593
  yield item
594
+
595
+ async def _gtorHistory(self):
596
+ return [HttpResp(hnfo) for hnfo in self.valu.get('history')]
@@ -877,7 +877,7 @@ class User(s_stormtypes.Prim):
877
877
  ''',
878
878
  'type': {'type': 'function', '_funcname': '_methUserSetRoles',
879
879
  'args': (
880
- {'name': 'idens', 'type': 'list', 'desc': 'The idens to of the Role.', },
880
+ {'name': 'idens', 'type': 'list', 'desc': 'The idens of the Roles to set on the User.', },
881
881
  ),
882
882
  'returns': {'type': 'null', }}},
883
883
  {'name': 'revoke', 'desc': 'Remove a Role from the User',
@@ -1085,7 +1085,7 @@ class CortexHttpApi(s_stormtypes.Lib):
1085
1085
  {'name': 'path', 'type': 'string',
1086
1086
  'desc': 'The extended HTTP API path.'},
1087
1087
  {'name': 'name', 'type': 'string',
1088
- 'desc': 'Friendly name for the Extended HTTP API', 'default': ''},
1088
+ 'desc': 'Friendly name for the Extended HTTP API.', 'default': ''},
1089
1089
  {'name': 'desc', 'type': 'string',
1090
1090
  'desc': 'Description for the Extended HTTP API.', 'default': ''},
1091
1091
  {'name': 'runas', 'type': 'string',
@@ -1095,6 +1095,8 @@ class CortexHttpApi(s_stormtypes.Lib):
1095
1095
  'desc': 'Require the API endpoint to be authenticated.', 'default': True},
1096
1096
  {'name': 'readonly', 'type': 'boolean',
1097
1097
  'desc': 'Run the Extended HTTP Storm methods in readonly mode.', 'default': False},
1098
+ {'name': 'iden', 'type': 'str',
1099
+ 'desc': 'An iden for the new Extended HTTP API.', 'default': None},
1098
1100
  ),
1099
1101
  'returns': {'type': 'http:api', 'desc': 'A new ``http:api`` object.'}}},
1100
1102
  {'name': 'del', 'desc': 'Delete an Extended HTTP API endpoint.',
@@ -1196,7 +1198,7 @@ class CortexHttpApi(s_stormtypes.Lib):
1196
1198
  apis = [HttpApi(self.runt, adef) for adef in adefs]
1197
1199
  return apis
1198
1200
 
1199
- async def addHttpApi(self, path, name='', desc='', runas='owner', authenticated=True, readonly=False):
1201
+ async def addHttpApi(self, path, name='', desc='', runas='owner', authenticated=True, readonly=False, iden=None):
1200
1202
  s_stormtypes.confirm(('storm', 'lib', 'cortex', 'httpapi', 'add'))
1201
1203
 
1202
1204
  path = await s_stormtypes.tostr(path)
@@ -1207,6 +1209,7 @@ class CortexHttpApi(s_stormtypes.Lib):
1207
1209
  authenticated = await s_stormtypes.tobool(authenticated)
1208
1210
 
1209
1211
  adef = {
1212
+ 'iden': iden,
1210
1213
  'path': path,
1211
1214
  'view': self.runt.snap.view.iden,
1212
1215
  'runas': runas,
@@ -1,11 +1,11 @@
1
1
  import logging
2
- import ipaddress
3
-
4
2
 
5
3
  import synapse.exc as s_exc
4
+ import synapse.common as s_common
6
5
 
7
6
  import synapse.lib.stormtypes as s_stormtypes
8
7
 
8
+ ipaddress = s_common.ipaddress
9
9
  logger = logging.getLogger(__name__)
10
10
 
11
11
  @s_stormtypes.registry.registerLib
@@ -3,6 +3,7 @@ import synapse.common as s_common
3
3
 
4
4
  import synapse.lib.node as s_node
5
5
  import synapse.lib.cache as s_cache
6
+ import synapse.lib.layer as s_layer
6
7
  import synapse.lib.stormtypes as s_stormtypes
7
8
 
8
9
  import synapse.models.infotech as s_infotech
@@ -497,6 +498,7 @@ class ModelType(s_stormtypes.Prim):
497
498
  _storm_locals = (
498
499
  {'name': 'name', 'desc': 'The name of the Type.', 'type': 'str', },
499
500
  {'name': 'stortype', 'desc': 'The storetype of the Type.', 'type': 'int', },
501
+ {'name': 'opts', 'desc': 'The options for the Type.', 'type': 'dict', },
500
502
  {'name': 'repr', 'desc': 'Get the repr of a value for the Type.',
501
503
  'type': {'type': 'function', '_funcname': '_methRepr',
502
504
  'args': (
@@ -516,6 +518,7 @@ class ModelType(s_stormtypes.Prim):
516
518
  s_stormtypes.Prim.__init__(self, valu, path=path)
517
519
  self.locls.update(self.getObjLocals())
518
520
  self.locls.update({'name': valu.name,
521
+ 'opts': valu.opts,
519
522
  'stortype': valu.stortype,
520
523
  })
521
524
 
@@ -756,6 +759,16 @@ class MigrationEditorMixin:
756
759
  if overwrite or not proto.hasTagProp(tagname, propname):
757
760
  await proto.setTagProp(tagname, propname, valu) # use tag perms
758
761
 
762
+ async def copyExtProps(self, src, proto):
763
+
764
+ form = src.form
765
+
766
+ for name, valu in src.props.items():
767
+ prop = form.props.get(name)
768
+ if not prop.isext:
769
+ continue
770
+
771
+ await proto.set(name, valu)
759
772
 
760
773
  @s_stormtypes.registry.registerLib
761
774
  class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
@@ -788,6 +801,13 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
788
801
  'desc': 'Copy tag property value even if the property exists on the destination node.', },
789
802
  ),
790
803
  'returns': {'type': 'null', }}},
804
+ {'name': 'copyExtProps', 'desc': 'Copy extended properties from the src node to the dst node.',
805
+ 'type': {'type': 'function', '_funcname': '_methCopyExtProps',
806
+ 'args': (
807
+ {'name': 'src', 'type': 'node', 'desc': 'The node to copy extended props from.', },
808
+ {'name': 'dst', 'type': 'node', 'desc': 'The node to copy extended props to.', },
809
+ ),
810
+ 'returns': {'type': 'null', }}},
791
811
  )
792
812
  _storm_lib_path = ('model', 'migration')
793
813
 
@@ -796,6 +816,9 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
796
816
  'copyData': self._methCopyData,
797
817
  'copyEdges': self._methCopyEdges,
798
818
  'copyTags': self._methCopyTags,
819
+ 'copyExtProps': self._methCopyExtProps,
820
+ 'liftByPropValuNoNorm': self._methLiftByPropValuNoNorm,
821
+ 'setNodePropValuNoNorm': self._methSetNodePropValuNoNorm,
799
822
  }
800
823
 
801
824
  async def _methCopyData(self, src, dst, overwrite=False):
@@ -839,6 +862,79 @@ class LibModelMigration(s_stormtypes.Lib, MigrationEditorMixin):
839
862
  proto = editor.loadNode(dst)
840
863
  await self.copyTags(src, proto, overwrite=overwrite)
841
864
 
865
+ async def _methCopyExtProps(self, src, dst):
866
+
867
+ if not isinstance(src, s_node.Node):
868
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyExtProps() source argument must be a node.')
869
+ if not isinstance(dst, s_node.Node):
870
+ raise s_exc.BadArg(mesg='$lib.model.migration.copyExtProps() dest argument must be a node.')
871
+
872
+ snap = self.runt.snap
873
+
874
+ async with snap.getEditor() as editor:
875
+ proto = editor.loadNode(dst)
876
+ await self.copyExtProps(src, proto)
877
+
878
+ async def _methLiftByPropValuNoNorm(self, formname, propname, valu, cmpr='=', reverse=False):
879
+ '''
880
+ No storm docs for this on purpose. It is restricted for use during model migrations only.
881
+ '''
882
+ formname = await s_stormtypes.tostr(formname)
883
+ propname = await s_stormtypes.tostr(propname)
884
+ valu = await s_stormtypes.toprim(valu)
885
+
886
+ prop = self.runt.snap.core.model.prop(f'{formname}:{propname}')
887
+ if prop is None:
888
+ mesg = f'Could not find prop: {formname}:{propname}'
889
+ raise s_exc.NoSuchProp(mesg=mesg, formname=formname, propname=propname)
890
+
891
+ if not self.runt.snap.core.migration:
892
+ mesg = '$lib.model.migration.liftByPropValuNoNorm() is restricted to model migrations only.'
893
+ raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
894
+
895
+ stortype = prop.type.stortype
896
+
897
+ # Normally we'd call proptype.getStorCmprs() here to get the cmprvals
898
+ # but getStorCmprs() calls norm() which we're trying to avoid so build
899
+ # cmprvals manually here.
900
+
901
+ if prop.type.isarray:
902
+ stortype &= (~s_layer.STOR_FLAG_ARRAY)
903
+ liftfunc = self.runt.snap.wlyr.liftByPropArray
904
+ else:
905
+ liftfunc = self.runt.snap.wlyr.liftByPropValu
906
+
907
+ cmprvals = ((cmpr, valu, stortype),)
908
+
909
+ layriden = self.runt.snap.wlyr.iden
910
+ async for _, buid, sode in liftfunc(formname, propname, cmprvals, reverse=reverse):
911
+ yield await self.runt.snap._joinStorNode(buid, {layriden: sode})
912
+
913
+ async def _methSetNodePropValuNoNorm(self, n, propname, valu):
914
+ '''
915
+ No storm docs for this on purpose. It is restricted for use during model migrations only.
916
+ '''
917
+
918
+ # NB: I'm sure there are all kinds of edges cases that this function doesn't account for. At the time of it's
919
+ # creation, this was intended to be used to update array properties with bad it:sec:cpe values in them. It works
920
+ # for that use case (see model migration 0.2.28). Any additional use of this function should perform heavy
921
+ # testing.
922
+
923
+ if not isinstance(n, s_node.Node):
924
+ raise s_exc.BadArg(mesg='$lib.model.migration.setNodePropValuNoNorm() argument must be a node.')
925
+
926
+ if not self.runt.snap.core.migration:
927
+ mesg = '$lib.model.migration.setNodePropValuNoNorm() is restricted to model migrations only.'
928
+ raise s_exc.AuthDeny(mesg=mesg, user=self.runt.user.iden, username=self.runt.user.name)
929
+
930
+ propname = await s_stormtypes.tostr(propname)
931
+ valu = await s_stormtypes.toprim(valu)
932
+
933
+ async with self.runt.snap.getNodeEditor(n) as proto:
934
+ await proto.set(propname, valu, norminfo={})
935
+
936
+ return n
937
+
842
938
  @s_stormtypes.registry.registerLib
843
939
  class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
844
940
  '''
@@ -980,10 +1076,15 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
980
1076
  def getObjLocals(self):
981
1077
  return {
982
1078
  'itSecCpe_2_170_0': self._itSecCpe_2_170_0,
1079
+ 'itSecCpe_2_170_0_internal': self._itSecCpe_2_170_0_internal,
983
1080
  'riskHasVulnToVulnerable': self._riskHasVulnToVulnerable,
984
1081
  }
985
1082
 
986
1083
  async def _itSecCpe_2_170_0(self, n, prefer_v22=False, force=False):
1084
+ info = await self._itSecCpe_2_170_0_internal(n, prefer_v22=prefer_v22, force=force, set_nodedata=True)
1085
+ return info.get('status') == 'success'
1086
+
1087
+ async def _itSecCpe_2_170_0_internal(self, n, prefer_v22=False, force=False, set_nodedata=False):
987
1088
 
988
1089
  if not isinstance(n, s_node.Node):
989
1090
  raise s_exc.BadArg(mesg='$lib.model.migration.s.itSecCpe_2_170_0() argument must be a node.')
@@ -1008,13 +1109,12 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
1008
1109
  if self.runt.debug:
1009
1110
  mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node already migrated.'
1010
1111
  await self.runt.printf(mesg)
1011
- return True
1112
+ return nodedata
1012
1113
 
1013
1114
  modl = self.runt.model.type('it:sec:cpe')
1014
1115
 
1015
1116
  valu23 = None
1016
1117
  valu22 = None
1017
- invalid = ''
1018
1118
 
1019
1119
  # Check the primary property for validity.
1020
1120
  cpe23 = s_infotech.cpe23_regex.match(curv)
@@ -1036,11 +1136,11 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
1036
1136
  mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): Node is valid, no migration necessary.'
1037
1137
  await self.runt.printf(mesg)
1038
1138
 
1039
- await proto.setData('migration.s.itSecCpe_2_170_0', {
1040
- 'status': 'success',
1041
- })
1139
+ nodedata = {'status': 'success'}
1140
+ if set_nodedata:
1141
+ await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
1042
1142
 
1043
- return True
1143
+ return nodedata
1044
1144
 
1045
1145
  if valu23 is None and valu22 is None:
1046
1146
  reason = 'Unable to migrate due to invalid data. Primary property and :v2_2 are both invalid.'
@@ -1049,12 +1149,14 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
1049
1149
  mesg = f'itSecCpe_2_170_0({reprvalu}): {reason}'
1050
1150
  await self.runt.warn(mesg)
1051
1151
 
1052
- await proto.setData('migration.s.itSecCpe_2_170_0', {
1152
+ nodedata = {
1053
1153
  'status': 'failed',
1054
1154
  'reason': reason,
1055
- })
1155
+ }
1156
+ if set_nodedata:
1157
+ await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
1056
1158
 
1057
- return False
1159
+ return nodedata
1058
1160
 
1059
1161
  if prefer_v22:
1060
1162
  valu = valu22 or valu23
@@ -1065,7 +1167,6 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
1065
1167
  norm, info = modl.norm(valu)
1066
1168
  subs = info.get('subs')
1067
1169
 
1068
- edits = []
1069
1170
  nodedata = {'status': 'success'}
1070
1171
 
1071
1172
  if norm != curv:
@@ -1097,7 +1198,8 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
1097
1198
  # Update the existing property with the re-normalized property value.
1098
1199
  await proto.set(propname, subscurv, ignore_ro=True)
1099
1200
 
1100
- await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
1201
+ if set_nodedata:
1202
+ await proto.setData('migration.s.itSecCpe_2_170_0', nodedata)
1101
1203
 
1102
1204
  if self.runt.debug:
1103
1205
  if nodedata.get('updated'):
@@ -1107,7 +1209,7 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
1107
1209
  mesg = f'DEBUG: itSecCpe_2_170_0({reprvalu}): No property updates required.'
1108
1210
  await self.runt.printf(mesg)
1109
1211
 
1110
- return True
1212
+ return nodedata
1111
1213
 
1112
1214
  async def _riskHasVulnToVulnerable(self, n, nodata=False):
1113
1215
 
@@ -847,7 +847,7 @@ class LibProjects(s_stormtypes.Lib):
847
847
  {'name': 'name', 'type': 'str', 'desc': 'The name of the Project to get'},
848
848
  ),
849
849
  'returns': {'type': 'proj:project',
850
- 'desc': 'The `proj:project `object, if it exists, otherwise null'}}},
850
+ 'desc': 'The project object, if it exists, otherwise null'}}},
851
851
 
852
852
  {'name': 'add', 'desc': 'Add a new project',
853
853
  'type': {'type': 'function', '_funcname': '_funcProjAdd',