synapse 2.186.0__py311-none-any.whl → 2.188.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (58) hide show
  1. synapse/cortex.py +133 -9
  2. synapse/datamodel.py +20 -4
  3. synapse/exc.py +14 -1
  4. synapse/lib/ast.py +6 -4
  5. synapse/lib/auth.py +9 -0
  6. synapse/lib/hive.py +1 -1
  7. synapse/lib/httpapi.py +2 -1
  8. synapse/lib/modelrev.py +771 -11
  9. synapse/lib/nexus.py +6 -0
  10. synapse/lib/node.py +5 -3
  11. synapse/lib/scrape.py +18 -104
  12. synapse/lib/spooled.py +26 -3
  13. synapse/lib/storm.py +51 -28
  14. synapse/lib/stormlib/model.py +320 -250
  15. synapse/lib/stormlib/modelext.py +31 -0
  16. synapse/lib/stormlib/scrape.py +1 -4
  17. synapse/lib/stormtypes.py +53 -11
  18. synapse/lib/version.py +2 -2
  19. synapse/lib/view.py +9 -3
  20. synapse/models/base.py +27 -0
  21. synapse/models/files.py +22 -0
  22. synapse/models/inet.py +49 -4
  23. synapse/models/infotech.py +49 -22
  24. synapse/models/orgs.py +64 -2
  25. synapse/models/proj.py +1 -6
  26. synapse/models/risk.py +65 -0
  27. synapse/tests/test_cortex.py +21 -0
  28. synapse/tests/test_lib_agenda.py +13 -0
  29. synapse/tests/test_lib_auth.py +15 -0
  30. synapse/tests/test_lib_cell.py +2 -1
  31. synapse/tests/test_lib_httpapi.py +6 -0
  32. synapse/tests/test_lib_modelrev.py +918 -379
  33. synapse/tests/test_lib_nexus.py +26 -0
  34. synapse/tests/test_lib_scrape.py +14 -6
  35. synapse/tests/test_lib_spooled.py +34 -0
  36. synapse/tests/test_lib_storm.py +48 -0
  37. synapse/tests/test_lib_stormlib_model.py +0 -270
  38. synapse/tests/test_lib_stormlib_modelext.py +76 -1
  39. synapse/tests/test_lib_stormlib_scrape.py +0 -8
  40. synapse/tests/test_lib_stormtypes.py +12 -1
  41. synapse/tests/test_lib_trigger.py +8 -0
  42. synapse/tests/test_lib_view.py +24 -0
  43. synapse/tests/test_model_base.py +11 -0
  44. synapse/tests/test_model_files.py +19 -0
  45. synapse/tests/test_model_inet.py +33 -0
  46. synapse/tests/test_model_infotech.py +14 -11
  47. synapse/tests/test_model_orgs.py +39 -0
  48. synapse/tests/test_model_proj.py +11 -1
  49. synapse/tests/test_model_risk.py +32 -0
  50. synapse/tools/changelog.py +11 -3
  51. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/METADATA +1 -1
  52. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/RECORD +55 -58
  53. synapse/assets/__init__.py +0 -35
  54. synapse/assets/storm/migrations/model-0.2.28.storm +0 -355
  55. synapse/tests/test_assets.py +0 -25
  56. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/LICENSE +0 -0
  57. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/WHEEL +0 -0
  58. {synapse-2.186.0.dist-info → synapse-2.188.0.dist-info}/top_level.txt +0 -0
synapse/lib/stormtypes.py CHANGED
@@ -3816,13 +3816,13 @@ class Queue(StormType):
3816
3816
  'args': (
3817
3817
  {'name': 'item', 'type': 'prim', 'desc': 'The item being put into the queue.', },
3818
3818
  ),
3819
- 'returns': {'type': 'null', }}},
3819
+ 'returns': {'type': 'int', 'desc': 'The queue offset of the item.'}}},
3820
3820
  {'name': 'puts', 'desc': 'Put multiple items into the Queue.',
3821
3821
  'type': {'type': 'function', '_funcname': '_methQueuePuts',
3822
3822
  'args': (
3823
3823
  {'name': 'items', 'type': 'list', 'desc': 'The items to put into the Queue.', },
3824
3824
  ),
3825
- 'returns': {'type': 'null', }}},
3825
+ 'returns': {'type': 'int', 'desc': 'The queue offset of the first item.'}}},
3826
3826
  {'name': 'gets', 'desc': 'Get multiple items from the Queue as a iterator.',
3827
3827
  'type': {'type': 'function', '_funcname': '_methQueueGets',
3828
3828
  'args': (
@@ -6130,14 +6130,14 @@ class Node(Prim):
6130
6130
  'type': {'type': 'function', '_funcname': '_methNodeAddEdge',
6131
6131
  'args': (
6132
6132
  {'name': 'verb', 'type': 'str', 'desc': 'The edge verb to add.'},
6133
- {'name': 'iden', 'type': 'str', 'desc': 'The node id of the destination node.'},
6133
+ {'name': 'iden', 'type': 'str', 'desc': 'The node iden of the destination node.'},
6134
6134
  ),
6135
6135
  'returns': {'type': 'null', }}},
6136
6136
  {'name': 'delEdge', 'desc': 'Remove a light-weight edge.',
6137
6137
  'type': {'type': 'function', '_funcname': '_methNodeDelEdge',
6138
6138
  'args': (
6139
6139
  {'name': 'verb', 'type': 'str', 'desc': 'The edge verb to remove.'},
6140
- {'name': 'iden', 'type': 'str', 'desc': 'The node id of the destination node to remove.'},
6140
+ {'name': 'iden', 'type': 'str', 'desc': 'The node iden of the destination node to remove.'},
6141
6141
  ),
6142
6142
  'returns': {'type': 'null', }}},
6143
6143
  {'name': 'globtags', 'desc': 'Get a list of the tag components from a Node which match a tag glob expression.',
@@ -6827,11 +6827,11 @@ class Layer(Prim):
6827
6827
  'returns': {'name': 'Yields', 'type': 'list',
6828
6828
  'desc': 'Yields messages describing any index inconsistencies.', }}},
6829
6829
  {'name': 'getStorNode', 'desc': '''
6830
- Retrieve the raw storage node for the specified node id.
6830
+ Retrieve the raw storage node for the specified node iden.
6831
6831
  ''',
6832
6832
  'type': {'type': 'function', '_funcname': 'getStorNode',
6833
6833
  'args': (
6834
- {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node id.'},
6834
+ {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
6835
6835
  ),
6836
6836
  'returns': {'type': 'dict', 'desc': 'The storage node dictionary.', }}},
6837
6837
  {'name': 'liftByProp', 'desc': '''
@@ -6897,7 +6897,7 @@ class Layer(Prim):
6897
6897
  'desc': 'Yields (<n1iden>, <verb>, <n2iden>) tuples', }}},
6898
6898
 
6899
6899
  {'name': 'getEdgesByN1', 'desc': '''
6900
- Yield (verb, n2iden) tuples for any light edges in the layer for the source node id.
6900
+ Yield (verb, n2iden) tuples for any light edges in the layer for the source node iden.
6901
6901
 
6902
6902
  Example:
6903
6903
  Iterate the N1 edges for ``$node``::
@@ -6909,13 +6909,13 @@ class Layer(Prim):
6909
6909
  ''',
6910
6910
  'type': {'type': 'function', '_funcname': 'getEdgesByN1',
6911
6911
  'args': (
6912
- {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node id.'},
6912
+ {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
6913
6913
  ),
6914
6914
  'returns': {'name': 'Yields', 'type': 'list',
6915
6915
  'desc': 'Yields (<verb>, <n2iden>) tuples', }}},
6916
6916
 
6917
6917
  {'name': 'getEdgesByN2', 'desc': '''
6918
- Yield (verb, n1iden) tuples for any light edges in the layer for the target node id.
6918
+ Yield (verb, n1iden) tuples for any light edges in the layer for the target node iden.
6919
6919
 
6920
6920
  Example:
6921
6921
  Iterate the N2 edges for ``$node``::
@@ -6926,10 +6926,26 @@ class Layer(Prim):
6926
6926
  ''',
6927
6927
  'type': {'type': 'function', '_funcname': 'getEdgesByN2',
6928
6928
  'args': (
6929
- {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node id.'},
6929
+ {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
6930
6930
  ),
6931
6931
  'returns': {'name': 'Yields', 'type': 'list',
6932
6932
  'desc': 'Yields (<verb>, <n1iden>) tuples', }}},
6933
+ {'name': 'getNodeData', 'desc': '''
6934
+ Yield (name, valu) tuples for any node data in the layer for the target node iden.
6935
+
6936
+ Example:
6937
+ Iterate the node data for ``$node``::
6938
+
6939
+ for ($name, $valu) in $layer.getNodeData($node.iden()) {
6940
+ $lib.print(`{$name} = {$valu}`)
6941
+ }
6942
+ ''',
6943
+ 'type': {'type': 'function', '_funcname': 'getNodeData',
6944
+ 'args': (
6945
+ {'name': 'nodeid', 'type': 'str', 'desc': 'The hex string of the node iden.'},
6946
+ ),
6947
+ 'returns': {'name': 'Yields', 'type': 'list',
6948
+ 'desc': 'Yields (<name>, <valu>) tuples', }}},
6933
6949
  )
6934
6950
  _storm_typename = 'layer'
6935
6951
  _ismutable = False
@@ -6986,6 +7002,7 @@ class Layer(Prim):
6986
7002
  'getStorNodesByForm': self.getStorNodesByForm,
6987
7003
  'getEdgesByN1': self.getEdgesByN1,
6988
7004
  'getEdgesByN2': self.getEdgesByN2,
7005
+ 'getNodeData': self.getNodeData,
6989
7006
  'getMirrorStatus': self.getMirrorStatus,
6990
7007
  }
6991
7008
 
@@ -7355,6 +7372,15 @@ class Layer(Prim):
7355
7372
  async for item in layr.iterNodeEdgesN2(s_common.uhex(nodeid)):
7356
7373
  yield item
7357
7374
 
7375
+ @stormfunc(readonly=True)
7376
+ async def getNodeData(self, nodeid):
7377
+ nodeid = await tostr(nodeid)
7378
+ layriden = self.valu.get('iden')
7379
+ await self.runt.reqUserCanReadLayer(layriden)
7380
+ layr = self.runt.snap.core.getLayer(layriden)
7381
+ async for item in layr.iterNodeData(s_common.uhex(nodeid)):
7382
+ yield item
7383
+
7358
7384
  @stormfunc(readonly=True)
7359
7385
  async def _methLayerGet(self, name, defv=None):
7360
7386
  return self.valu.get(name, defv)
@@ -7519,6 +7545,9 @@ class View(Prim):
7519
7545
  {'name': 'parent', 'desc': 'The parent View. Will be ``$lib.null`` if the view is not a fork.', 'type': 'str'},
7520
7546
  {'name': 'triggers', 'desc': 'The ``trigger`` objects associated with the ``view``.',
7521
7547
  'type': 'list', },
7548
+ {'name': 'children', 'desc': 'Yield Views which are children of this View.',
7549
+ 'type': {'type': 'function', '_funcname': '_methGetChildren',
7550
+ 'returns': {'name': 'yields', 'type': 'view', 'desc': 'Child Views.', }}},
7522
7551
  {'name': 'set', 'desc': '''
7523
7552
  Set a view configuration option.
7524
7553
 
@@ -7806,6 +7835,7 @@ class View(Prim):
7806
7835
  'merge': self._methViewMerge,
7807
7836
  'detach': self.detach,
7808
7837
  'addNode': self.addNode,
7838
+ 'children': self._methGetChildren,
7809
7839
  'getEdges': self._methGetEdges,
7810
7840
  'wipeLayer': self._methWipeLayer,
7811
7841
  'swapLayer': self._methSwapLayer,
@@ -7941,6 +7971,12 @@ class View(Prim):
7941
7971
  async for valu in view.iterPropValues(propname):
7942
7972
  yield valu
7943
7973
 
7974
+ @stormfunc(readonly=True)
7975
+ async def _methGetChildren(self):
7976
+ view = self._reqView()
7977
+ async for child in view.children():
7978
+ yield View(self.runt, await child.pack(), path=self.path)
7979
+
7944
7980
  @stormfunc(readonly=True)
7945
7981
  async def _methGetEdges(self, verb=None):
7946
7982
  verb = await toprim(verb)
@@ -9584,7 +9620,7 @@ def fromprim(valu, path=None, basetypes=True):
9584
9620
 
9585
9621
  return valu
9586
9622
 
9587
- async def tostor(valu):
9623
+ async def tostor(valu, isndef=False):
9588
9624
 
9589
9625
  if isinstance(valu, Number):
9590
9626
  return str(valu.value())
@@ -9607,6 +9643,9 @@ async def tostor(valu):
9607
9643
  pass
9608
9644
  return retn
9609
9645
 
9646
+ if isndef and isinstance(valu, s_node.Node):
9647
+ return valu.ndef
9648
+
9610
9649
  return await toprim(valu)
9611
9650
 
9612
9651
  async def tocmprvalu(valu):
@@ -9650,6 +9689,9 @@ async def tostr(valu, noneok=False):
9650
9689
  if isinstance(valu, bytes):
9651
9690
  return valu.decode('utf8', 'surrogatepass')
9652
9691
 
9692
+ if isinstance(valu, s_node.Node):
9693
+ return valu.repr()
9694
+
9653
9695
  return str(valu)
9654
9696
  except Exception as e:
9655
9697
  mesg = f'Failed to make a string from {valu!r}.'
synapse/lib/version.py CHANGED
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
223
223
  ##############################################################################
224
224
  # The following are touched during the release process by bumpversion.
225
225
  # Do not modify these directly.
226
- version = (2, 186, 0)
226
+ version = (2, 188, 0)
227
227
  verstring = '.'.join([str(x) for x in version])
228
- commit = 'ffa4abc52513d4b8ac592496322227eed9e07dc5'
228
+ commit = '6d1f734406234e3750c72f5bf299f89a13d0825a'
synapse/lib/view.py CHANGED
@@ -1315,6 +1315,14 @@ class View(s_nexus.Pusher): # type: ignore
1315
1315
 
1316
1316
  todo.append(child)
1317
1317
 
1318
+ async def children(self):
1319
+ for view in list(self.core.views.values()):
1320
+ if view.parent != self:
1321
+ await asyncio.sleep(0)
1322
+ continue
1323
+
1324
+ yield view
1325
+
1318
1326
  async def insertParentFork(self, useriden, name=None):
1319
1327
  '''
1320
1328
  Insert a new View between a forked View and its parent.
@@ -1614,9 +1622,7 @@ class View(s_nexus.Pusher): # type: ignore
1614
1622
  if trig is not None:
1615
1623
  return self.triggers.get(tdef['iden']).pack()
1616
1624
 
1617
- gate = self.core.auth.getAuthGate(tdef['iden'])
1618
- if gate is not None:
1619
- raise s_exc.DupIden(mesg='An AuthGate with this iden already exists')
1625
+ self.core.auth.reqNoAuthGate(tdef['iden'])
1620
1626
 
1621
1627
  user = self.core.auth.user(tdef['user'])
1622
1628
  await self.core.getStormQuery(tdef['storm'])
synapse/models/base.py CHANGED
@@ -107,6 +107,19 @@ class BaseModule(s_module.CoreModule):
107
107
 
108
108
  ('meta:sophistication', ('int', {'enums': sophenums}), {
109
109
  'doc': 'A sophistication score with named values: very low, low, medium, high, and very high.'}),
110
+
111
+ ('meta:aggregate:type:taxonomy', ('taxonomy', {}), {
112
+ 'interfaces': ('meta:taxonomy',),
113
+ 'doc': 'A type of item being counted in aggregate.'}),
114
+
115
+ ('meta:aggregate', ('guid', {}), {
116
+ 'display': {
117
+ 'columns': (
118
+ {'type': 'prop', 'opts': {'name': 'type'}},
119
+ {'type': 'prop', 'opts': {'name': 'count'}},
120
+ ),
121
+ },
122
+ 'doc': 'A node which represents an aggregate count of a specific type.'}),
110
123
  ),
111
124
  'interfaces': (
112
125
  ('meta:taxonomy', {
@@ -285,6 +298,20 @@ class BaseModule(s_module.CoreModule):
285
298
  'doc': 'An external identifier for the rule.'}),
286
299
  )),
287
300
 
301
+ ('meta:aggregate:type:taxonomy', {}, ()),
302
+ ('meta:aggregate', {}, (
303
+
304
+ ('type', ('meta:aggregate:type:taxonomy', {}), {
305
+ 'ex': 'casualties.civilian',
306
+ 'doc': 'The type of items being counted in aggregate.'}),
307
+
308
+ ('time', ('time', {}), {
309
+ 'doc': 'The time that the count was computed.'}),
310
+
311
+ ('count', ('int', {}), {
312
+ 'doc': 'The number of items counted in aggregate.'}),
313
+ )),
314
+
288
315
  ('graph:cluster', {}, (
289
316
  ('name', ('str', {'lower': True}), {
290
317
  'doc': 'A human friendly name for the cluster.'}),
synapse/models/files.py CHANGED
@@ -284,6 +284,16 @@ class FileModule(s_module.CoreModule):
284
284
  'doc': 'A parent file that fully contains the specified child file.',
285
285
  }),
286
286
 
287
+ ('file:attachment', ('guid', {}), {
288
+ 'display': {
289
+ 'columns': (
290
+ {'type': 'prop', 'opts': {'name': 'name'}},
291
+ {'type': 'prop', 'opts': {'name': 'file'}},
292
+ {'type': 'prop', 'opts': {'name': 'text'}},
293
+ ),
294
+ },
295
+ 'doc': 'A file attachment.'}),
296
+
287
297
  ('file:archive:entry', ('guid', {}), {
288
298
  'doc': 'An archive entry representing a file and metadata within a parent archive file.'}),
289
299
 
@@ -601,6 +611,18 @@ class FileModule(s_module.CoreModule):
601
611
  }),
602
612
  )),
603
613
 
614
+ ('file:attachment', {}, (
615
+
616
+ ('name', ('file:path', {}), {
617
+ 'doc': 'The name of the attached file.'}),
618
+
619
+ ('text', ('str', {}), {
620
+ 'doc': 'Any text associated with the file such as alt-text for images.'}),
621
+
622
+ ('file', ('file:bytes', {}), {
623
+ 'doc': 'The file which was attached.'}),
624
+ )),
625
+
604
626
  ('file:archive:entry', {}, (
605
627
 
606
628
  ('parent', ('file:bytes', {}), {
synapse/models/inet.py CHANGED
@@ -1369,6 +1369,9 @@ class InetModule(s_module.CoreModule):
1369
1369
  'doc': 'A username string.'
1370
1370
  }),
1371
1371
 
1372
+ ('inet:service:object', ('ndef', {'interfaces': ('inet:service:object',)}), {
1373
+ 'doc': 'An ndef type including all forms which implement the inet:service:object interface.'}),
1374
+
1372
1375
  ('inet:search:query', ('guid', {}), {
1373
1376
  'interfaces': ('inet:service:action',),
1374
1377
  'doc': 'An instance of a search query issued to a search engine.',
@@ -1530,6 +1533,14 @@ class InetModule(s_module.CoreModule):
1530
1533
  'interfaces': ('inet:service:object',),
1531
1534
  'doc': 'An account within a service platform. Accounts may be instance specific.'}),
1532
1535
 
1536
+ ('inet:service:relationship:type:taxonomy', ('taxonomy', {}), {
1537
+ 'interfaces': ('meta:taxonomy',),
1538
+ 'doc': 'A service object relationship type taxonomy.'}),
1539
+
1540
+ ('inet:service:relationship', ('guid', {}), {
1541
+ 'interfaces': ('inet:service:object',),
1542
+ 'doc': 'A relationship between two service objects.'}),
1543
+
1533
1544
  ('inet:service:permission:type:taxonomy', ('taxonomy', {}), {
1534
1545
  'interfaces': ('meta:taxonomy',),
1535
1546
  'doc': 'A permission type taxonomy.'}),
@@ -1588,6 +1599,10 @@ class InetModule(s_module.CoreModule):
1588
1599
  'interfaces': ('meta:taxonomy',),
1589
1600
  'doc': 'A message type taxonomy.'}),
1590
1601
 
1602
+ ('inet:service:emote', ('guid', {}), {
1603
+ 'interfaces': ('inet:service:object',),
1604
+ 'doc': 'An emote or reaction by an account.'}),
1605
+
1591
1606
  ('inet:service:access', ('guid', {}), {
1592
1607
  'interfaces': ('inet:service:action',),
1593
1608
  'doc': 'Represents a user access request to a service resource.'}),
@@ -2015,8 +2030,11 @@ class InetModule(s_module.CoreModule):
2015
2030
  'doc': 'The guid of the destination process.'
2016
2031
  }),
2017
2032
  ('dst:exe', ('file:bytes', {}), {
2018
- 'doc': 'The file (executable) that received the connection.'
2019
- }),
2033
+ 'doc': 'The file (executable) that received the connection.'}),
2034
+
2035
+ ('dst:txfiles', ('array', {'type': 'file:attachment', 'sorted': True, 'uniq': True}), {
2036
+ 'doc': 'An array of files sent by the destination host.'}),
2037
+
2020
2038
  ('dst:txcount', ('int', {}), {
2021
2039
  'doc': 'The number of packets sent by the destination host.'
2022
2040
  }),
@@ -2049,8 +2067,11 @@ class InetModule(s_module.CoreModule):
2049
2067
  'doc': 'The guid of the source process.'
2050
2068
  }),
2051
2069
  ('src:exe', ('file:bytes', {}), {
2052
- 'doc': 'The file (executable) that created the connection.'
2053
- }),
2070
+ 'doc': 'The file (executable) that created the connection.'}),
2071
+
2072
+ ('src:txfiles', ('array', {'type': 'file:attachment', 'sorted': True, 'uniq': True}), {
2073
+ 'doc': 'An array of files sent by the source host.'}),
2074
+
2054
2075
  ('src:txcount', ('int', {}), {
2055
2076
  'doc': 'The number of packets sent by the source host.'
2056
2077
  }),
@@ -3586,6 +3607,20 @@ class InetModule(s_module.CoreModule):
3586
3607
  'doc': 'Current profile details associated with the account.'}),
3587
3608
  )),
3588
3609
 
3610
+ ('inet:service:relationship:type:taxonomy', {}, ()),
3611
+ ('inet:service:relationship', {}, (
3612
+
3613
+ ('source', ('inet:service:object', {}), {
3614
+ 'doc': 'The source object.'}),
3615
+
3616
+ ('target', ('inet:service:object', {}), {
3617
+ 'doc': 'The target object.'}),
3618
+
3619
+ ('type', ('inet:service:relationship:type:taxonomy', {}), {
3620
+ 'ex': 'follows',
3621
+ 'doc': 'The type of relationship between the source and the target.'}),
3622
+ )),
3623
+
3589
3624
  ('inet:service:group', {}, ( # inet:service:object
3590
3625
 
3591
3626
  ('id', ('str', {'strip': True}), {
@@ -3743,6 +3778,16 @@ class InetModule(s_module.CoreModule):
3743
3778
  'doc': 'The file which was attached to the message.'}),
3744
3779
  )),
3745
3780
 
3781
+ ('inet:service:emote', {}, (
3782
+
3783
+ ('about', ('inet:service:object', {}), {
3784
+ 'doc': 'The node that the emote is about.'}),
3785
+
3786
+ ('text', ('str', {'strip': True}), {
3787
+ 'ex': ':partyparrot:',
3788
+ 'doc': 'The unicode or emote text of the reaction.'}),
3789
+ )),
3790
+
3746
3791
  ('inet:service:channel', {}, (
3747
3792
 
3748
3793
  ('name', ('str', {'onespace': True, 'lower': True}), {
@@ -18,10 +18,10 @@ import synapse.lib.version as s_version
18
18
 
19
19
  logger = logging.getLogger(__name__)
20
20
 
21
- # This is the regular expression pattern for CPE2.2. It's kind of a hybrid
21
+ # This is the regular expression pattern for CPE 2.2. It's kind of a hybrid
22
22
  # between compatible binding and preferred binding. Differences are here:
23
23
  # - Use only the list of percent encoded values specified by preferred binding.
24
- # This is to ensure it converts properly to CPE2.3.
24
+ # This is to ensure it converts properly to CPE 2.3.
25
25
  # - Add tilde (~) to the UNRESERVED list which removes the need to specify the
26
26
  # PACKED encoding specifically.
27
27
  ALPHA = '[A-Za-z]'
@@ -60,6 +60,14 @@ COMPONENT_LIST = f'''
60
60
  cpe22_regex = regex.compile(f'cpe:/{COMPONENT_LIST}', regex.VERBOSE | regex.IGNORECASE)
61
61
  cpe23_regex = regex.compile(s_scrape._cpe23_regex, regex.VERBOSE | regex.IGNORECASE)
62
62
 
63
+ def isValidCpe22(text):
64
+ rgx = cpe22_regex.fullmatch(text)
65
+ return rgx is not None
66
+
67
+ def isValidCpe23(text):
68
+ rgx = cpe23_regex.fullmatch(text)
69
+ return rgx is not None
70
+
63
71
  def cpesplit(text):
64
72
  part = ''
65
73
  parts = []
@@ -251,9 +259,20 @@ class Cpe22Str(s_types.Str):
251
259
  def _normPyStr(self, valu):
252
260
 
253
261
  text = valu.lower()
262
+
254
263
  if text.startswith('cpe:/'):
264
+
265
+ if not isValidCpe22(text):
266
+ mesg = 'CPE 2.2 string appears to be invalid.'
267
+ raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
268
+
255
269
  parts = chopCpe22(text)
256
270
  elif text.startswith('cpe:2.3:'):
271
+
272
+ if not isValidCpe23(text):
273
+ mesg = 'CPE 2.3 string appears to be invalid.'
274
+ raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
275
+
257
276
  parts = cpesplit(text[8:])
258
277
  else:
259
278
  mesg = 'CPE 2.2 string is expected to start with "cpe:/"'
@@ -261,8 +280,7 @@ class Cpe22Str(s_types.Str):
261
280
 
262
281
  v2_2 = zipCpe22(parts)
263
282
 
264
- rgx = cpe22_regex.match(v2_2)
265
- if rgx is None or rgx.group() != v2_2:
283
+ if not isValidCpe22(v2_2): # pragma: no cover
266
284
  mesg = 'CPE 2.2 string appears to be invalid.'
267
285
  raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
268
286
 
@@ -330,6 +348,12 @@ class Cpe23Str(s_types.Str):
330
348
  def _normPyStr(self, valu):
331
349
  text = valu.lower()
332
350
  if text.startswith('cpe:2.3:'):
351
+
352
+ # Validate the CPE 2.3 string immediately
353
+ if not isValidCpe23(text):
354
+ mesg = 'CPE 2.3 string appears to be invalid.'
355
+ raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
356
+
333
357
  parts = cpesplit(text[8:])
334
358
  if len(parts) > 11:
335
359
  mesg = f'CPE 2.3 string has {len(parts)} fields, expected up to 11.'
@@ -346,6 +370,10 @@ class Cpe23Str(s_types.Str):
346
370
  v2_2[idx] = ''
347
371
  continue
348
372
 
373
+ if idx in (PART_IDX_PART, PART_IDX_LANG) and part == '-':
374
+ v2_2[idx] = ''
375
+ continue
376
+
349
377
  part = fsb_unescape(part)
350
378
  v2_2[idx] = uri_quote(part)
351
379
 
@@ -357,12 +385,22 @@ class Cpe23Str(s_types.Str):
357
385
  v2_2[PART_IDX_OTHER]
358
386
  )
359
387
 
360
- v2_2 = v2_2[:7]
388
+ v2_2 = zipCpe22(v2_2[:7])
389
+
390
+ # Now validate the downconvert
391
+ if not isValidCpe22(v2_2): # pragma: no cover
392
+ mesg = 'Invalid CPE 2.3 to CPE 2.2 conversion.'
393
+ raise s_exc.BadTypeValu(mesg=mesg, valu=valu, v2_2=v2_2)
361
394
 
362
395
  parts = [fsb_unescape(k) for k in parts]
363
396
 
364
397
  elif text.startswith('cpe:/'):
365
398
 
399
+ # Validate the CPE 2.2 string immediately
400
+ if not isValidCpe22(text):
401
+ mesg = 'CPE 2.2 string appears to be invalid.'
402
+ raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
403
+
366
404
  v2_2 = text
367
405
  # automatically normalize CPE 2.2 format to CPE 2.3
368
406
  parts = chopCpe22(text)
@@ -408,24 +446,15 @@ class Cpe23Str(s_types.Str):
408
446
 
409
447
  v2_3 = 'cpe:2.3:' + ':'.join(escaped)
410
448
 
449
+ # Now validate the upconvert
450
+ if not isValidCpe23(v2_3): # pragma: no cover
451
+ mesg = 'Invalid CPE 2.2 to CPE 2.3 conversion.'
452
+ raise s_exc.BadTypeValu(mesg=mesg, valu=valu, v2_3=v2_3)
453
+
411
454
  else:
412
455
  mesg = 'CPE 2.3 string is expected to start with "cpe:2.3:"'
413
456
  raise s_exc.BadTypeValu(valu=valu, mesg=mesg)
414
457
 
415
- rgx = cpe23_regex.match(v2_3)
416
- if rgx is None or rgx.group() != v2_3:
417
- mesg = 'CPE 2.3 string appears to be invalid.'
418
- raise s_exc.BadTypeValu(mesg=mesg, valu=valu)
419
-
420
- if isinstance(v2_2, list):
421
- cpe22 = zipCpe22(v2_2)
422
- else:
423
- cpe22 = v2_2
424
-
425
- rgx = cpe22_regex.match(cpe22)
426
- if rgx is None or rgx.group() != cpe22:
427
- v2_2 = None
428
-
429
458
  subs = {
430
459
  'part': parts[PART_IDX_PART],
431
460
  'vendor': parts[PART_IDX_VENDOR],
@@ -438,11 +467,9 @@ class Cpe23Str(s_types.Str):
438
467
  'target_sw': parts[PART_IDX_TARGET_SW],
439
468
  'target_hw': parts[PART_IDX_TARGET_HW],
440
469
  'other': parts[PART_IDX_OTHER],
470
+ 'v2_2': v2_2,
441
471
  }
442
472
 
443
- if v2_2 is not None:
444
- subs['v2_2'] = v2_2
445
-
446
473
  return v2_3, {'subs': subs}
447
474
 
448
475
  class SemVer(s_types.Int):
synapse/models/orgs.py CHANGED
@@ -249,8 +249,24 @@ class OuModule(s_module.CoreModule):
249
249
  'doc': 'Vital statistics about an org for a given time period.',
250
250
  }),
251
251
  ('ou:opening', ('guid', {}), {
252
- 'doc': 'A job/work opening within an org.',
253
- }),
252
+ 'doc': 'A job/work opening within an org.'}),
253
+
254
+ ('ou:candidate:method:taxonomy', ('taxonomy', {}), {
255
+ 'interfaces': ('meta:taxonomy',),
256
+ 'doc': 'A taxonomy of methods by which a candidate came under consideration.'}),
257
+
258
+ ('ou:candidate', ('guid', {}), {
259
+ 'doc': 'A candidate being considered for a role within an organization.',
260
+ 'display': {
261
+ 'columns': (
262
+ {'type': 'prop', 'opts': {'name': 'contact::name'}},
263
+ {'type': 'prop', 'opts': {'name': 'contact::email'}},
264
+ {'type': 'prop', 'opts': {'name': 'submitted'}},
265
+ {'type': 'prop', 'opts': {'name': 'org::name'}},
266
+ {'type': 'prop', 'opts': {'name': 'opening::jobtitle'}},
267
+ ),
268
+ }}),
269
+
254
270
  ('ou:jobtype', ('taxonomy', {}), {
255
271
  'ex': 'it.dev.python',
256
272
  'doc': 'A taxonomy of job types.',
@@ -287,6 +303,8 @@ class OuModule(s_module.CoreModule):
287
303
  'doc': 'The campaign used the technique.'}),
288
304
  (('ou:org', 'uses', 'ou:technique'), {
289
305
  'doc': 'The org uses the technique.'}),
306
+ (('risk:vuln', 'uses', 'ou:technique'), {
307
+ 'doc': 'The vulnerability uses the technique.'}),
290
308
 
291
309
  (('ou:org', 'uses', None), {
292
310
  'doc': 'The ou:org makes use of the target node.'}),
@@ -354,6 +372,44 @@ class OuModule(s_module.CoreModule):
354
372
  }),
355
373
  # TODO a way to encode/normalize requirements.
356
374
  )),
375
+ ('ou:candidate:method:taxonomy', {}, ()),
376
+ ('ou:candidate', {}, (
377
+
378
+ ('org', ('ou:org', {}), {
379
+ 'doc': 'The organization considering the candidate.'}),
380
+
381
+ ('contact', ('ps:contact', {}), {
382
+ 'doc': 'The contact information of the candidate.'}),
383
+
384
+ ('method', ('ou:candidate:method:taxonomy', {}), {
385
+ 'doc': 'The method by which the candidate came under consideration.'}),
386
+
387
+ ('submitted', ('time', {}), {
388
+ 'doc': 'The time the candidate was submitted for consideration.'}),
389
+
390
+ ('intro', ('str', {'strip': True}), {
391
+ 'doc': 'An introduction or cover letter text submitted by the candidate.'}),
392
+
393
+ ('resume', ('file:bytes', {}), {
394
+ 'doc': "The candidate's resume or CV."}),
395
+
396
+ ('opening', ('ou:opening', {}), {
397
+ 'doc': 'The opening that the candidate is being considered for.'}),
398
+
399
+ ('agent', ('ps:contact', {}), {
400
+ 'doc': 'The contact information of an agent who advocates for the candidate.'}),
401
+
402
+ ('recruiter', ('ps:contact', {}), {
403
+ 'doc': 'The contact information of a recruiter who works on behalf of the organization.'}),
404
+
405
+ ('attachments', ('array', {'type': 'file:attachment', 'sorted': True, 'uniq': True}), {
406
+ 'doc': 'An array of additional files submitted by the candidate.'}),
407
+
408
+ # TODO: doc:questionare / responses
409
+ # TODO: :skills=[<ps:skill>]? vs :contact -> ps:proficiency?
410
+ # TODO: proj:task to track evaluation of the candidate?
411
+
412
+ )),
357
413
  ('ou:vitals', {}, (
358
414
 
359
415
  ('asof', ('time', {}), {
@@ -876,6 +932,12 @@ class OuModule(s_module.CoreModule):
876
932
  ('names', ('array', {'type': 'ou:industryname', 'uniq': True, 'sorted': True}), {
877
933
  'doc': 'An array of alternative names for the industry.'}),
878
934
 
935
+ ('reporter', ('ou:org', {}), {
936
+ 'doc': 'The organization reporting on the industry.'}),
937
+
938
+ ('reporter:name', ('ou:name', {}), {
939
+ 'doc': 'The name of the organization reporting on the industry.'}),
940
+
879
941
  ('subs', ('array', {'type': 'ou:industry', 'split': ',', 'uniq': True, 'sorted': True}), {
880
942
  'deprecated': True,
881
943
  'doc': 'Deprecated. Please use ou:industry:type taxonomy.'}),
synapse/models/proj.py CHANGED
@@ -16,10 +16,9 @@ class ProjectModule(s_module.CoreModule):
16
16
 
17
17
  async def initCoreModule(self):
18
18
  self.model.form('proj:project').onAdd(self._onAddProj)
19
- self.model.form('proj:project').onDel(self._onDelProj)
20
19
 
21
20
  async def _onAddProj(self, node):
22
- # ref counts on authgates?
21
+ # TODO: remove all storm:project authgates in 3.x migration
23
22
  gateiden = node.ndef[1]
24
23
 
25
24
  await self.core.auth.addAuthGate(node.ndef[1], 'storm:project')
@@ -29,10 +28,6 @@ class ProjectModule(s_module.CoreModule):
29
28
  rule = (True, ('project', 'admin'))
30
29
  await node.snap.user.addRule(rule, gateiden=gateiden)
31
30
 
32
- async def _onDelProj(self, node):
33
- gateiden = node.ndef[1]
34
- await self.core.auth.delAuthGate(gateiden)
35
-
36
31
  def getModelDefs(self):
37
32
  return (
38
33