synapse 2.176.0__py311-none-any.whl → 2.178.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (95) hide show
  1. synapse/axon.py +24 -9
  2. synapse/cortex.py +337 -172
  3. synapse/cryotank.py +46 -37
  4. synapse/datamodel.py +17 -4
  5. synapse/exc.py +19 -0
  6. synapse/lib/agenda.py +7 -13
  7. synapse/lib/aha.py +361 -88
  8. synapse/lib/auth.py +1520 -0
  9. synapse/lib/base.py +27 -9
  10. synapse/lib/cell.py +422 -163
  11. synapse/lib/config.py +15 -11
  12. synapse/lib/coro.py +13 -0
  13. synapse/lib/grammar.py +5 -0
  14. synapse/lib/hive.py +24 -3
  15. synapse/lib/hiveauth.py +6 -32
  16. synapse/lib/layer.py +7 -9
  17. synapse/lib/link.py +22 -18
  18. synapse/lib/lmdbslab.py +152 -3
  19. synapse/lib/modelrev.py +1 -1
  20. synapse/lib/nexus.py +24 -12
  21. synapse/lib/schemas.py +136 -0
  22. synapse/lib/storm.py +61 -29
  23. synapse/lib/stormlib/aha.py +1 -1
  24. synapse/lib/stormlib/auth.py +185 -10
  25. synapse/lib/stormlib/cortex.py +16 -5
  26. synapse/lib/stormlib/gen.py +80 -0
  27. synapse/lib/stormlib/imap.py +6 -2
  28. synapse/lib/stormlib/model.py +55 -0
  29. synapse/lib/stormlib/modelext.py +60 -0
  30. synapse/lib/stormlib/smtp.py +12 -2
  31. synapse/lib/stormlib/tabular.py +212 -0
  32. synapse/lib/stormtypes.py +14 -1
  33. synapse/lib/trigger.py +1 -1
  34. synapse/lib/version.py +2 -2
  35. synapse/lib/view.py +55 -28
  36. synapse/models/base.py +7 -0
  37. synapse/models/biz.py +4 -0
  38. synapse/models/files.py +8 -1
  39. synapse/models/inet.py +8 -0
  40. synapse/telepath.py +32 -17
  41. synapse/tests/files/aha/certs/cas/synapse.crt +28 -0
  42. synapse/tests/files/aha/certs/cas/synapse.key +51 -0
  43. synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt +30 -0
  44. synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key +51 -0
  45. synapse/tests/files/aha/certs/users/root@synapse.crt +29 -0
  46. synapse/tests/files/aha/certs/users/root@synapse.key +51 -0
  47. synapse/tests/files/changelog/model_2.176.0_16ee721a6b7221344eaf946c3ab4602dda546b1a.yaml.gz +0 -0
  48. synapse/tests/files/changelog/model_2.176.0_2a25c58bbd344716cd7cbc3f4304d8925b0f4ef2.yaml.gz +0 -0
  49. synapse/tests/files/rstorm/testsvc.py +1 -1
  50. synapse/tests/test_axon.py +8 -5
  51. synapse/tests/test_cortex.py +149 -141
  52. synapse/tests/test_cryotank.py +4 -4
  53. synapse/tests/test_datamodel.py +7 -0
  54. synapse/tests/test_lib_agenda.py +10 -3
  55. synapse/tests/test_lib_aha.py +336 -490
  56. synapse/tests/{test_lib_hiveauth.py → test_lib_auth.py} +314 -11
  57. synapse/tests/test_lib_base.py +20 -0
  58. synapse/tests/test_lib_cell.py +210 -30
  59. synapse/tests/test_lib_config.py +4 -3
  60. synapse/tests/test_lib_httpapi.py +18 -14
  61. synapse/tests/test_lib_layer.py +33 -33
  62. synapse/tests/test_lib_link.py +42 -1
  63. synapse/tests/test_lib_lmdbslab.py +68 -0
  64. synapse/tests/test_lib_nexus.py +12 -4
  65. synapse/tests/test_lib_node.py +0 -7
  66. synapse/tests/test_lib_storm.py +45 -0
  67. synapse/tests/test_lib_stormlib_aha.py +35 -36
  68. synapse/tests/test_lib_stormlib_auth.py +21 -0
  69. synapse/tests/test_lib_stormlib_cell.py +4 -15
  70. synapse/tests/test_lib_stormlib_cortex.py +12 -12
  71. synapse/tests/test_lib_stormlib_gen.py +99 -0
  72. synapse/tests/test_lib_stormlib_imap.py +14 -3
  73. synapse/tests/test_lib_stormlib_model.py +108 -0
  74. synapse/tests/test_lib_stormlib_modelext.py +64 -0
  75. synapse/tests/test_lib_stormlib_smtp.py +51 -0
  76. synapse/tests/test_lib_stormlib_tabular.py +226 -0
  77. synapse/tests/test_lib_stormsvc.py +4 -1
  78. synapse/tests/test_lib_stormtypes.py +10 -0
  79. synapse/tests/test_model_base.py +3 -0
  80. synapse/tests/test_model_biz.py +3 -0
  81. synapse/tests/test_model_files.py +12 -2
  82. synapse/tests/test_model_inet.py +24 -0
  83. synapse/tests/test_tools_aha.py +78 -101
  84. synapse/tests/test_tools_changelog.py +196 -0
  85. synapse/tests/test_tools_healthcheck.py +4 -3
  86. synapse/tests/utils.py +87 -121
  87. synapse/tools/aha/clone.py +50 -0
  88. synapse/tools/aha/enroll.py +2 -1
  89. synapse/tools/backup.py +2 -2
  90. synapse/tools/changelog.py +776 -15
  91. {synapse-2.176.0.dist-info → synapse-2.178.0.dist-info}/METADATA +48 -48
  92. {synapse-2.176.0.dist-info → synapse-2.178.0.dist-info}/RECORD +95 -82
  93. {synapse-2.176.0.dist-info → synapse-2.178.0.dist-info}/WHEEL +1 -1
  94. {synapse-2.176.0.dist-info → synapse-2.178.0.dist-info}/LICENSE +0 -0
  95. {synapse-2.176.0.dist-info → synapse-2.178.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py CHANGED
@@ -92,6 +92,7 @@ import synapse.lib.stormlib.scrape as s_stormlib_scrape # NOQA
92
92
  import synapse.lib.stormlib.infosec as s_stormlib_infosec # NOQA
93
93
  import synapse.lib.stormlib.project as s_stormlib_project # NOQA
94
94
  import synapse.lib.stormlib.spooled as s_stormlib_spooled # NOQA
95
+ import synapse.lib.stormlib.tabular as s_stormlib_tabular # NOQA
95
96
  import synapse.lib.stormlib.version as s_stormlib_version # NOQA
96
97
  import synapse.lib.stormlib.easyperm as s_stormlib_easyperm # NOQA
97
98
  import synapse.lib.stormlib.ethereum as s_stormlib_ethereum # NOQA
@@ -773,13 +774,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
773
774
  },
774
775
  'layer:lmdb:map_async': {
775
776
  'default': True,
776
- 'description': 'Set the default lmdb:map_async value in LMDB layers.',
777
- 'type': 'boolean'
777
+ 'description': 'Deprecated. This value is ignored.',
778
+ 'type': 'boolean',
779
+ 'hidecmdl': True,
780
+ 'hideconf': True,
778
781
  },
779
782
  'layer:lmdb:max_replay_log': {
780
783
  'default': 10000,
781
- 'description': 'Set the max size of the replay log for all layers.',
782
- 'type': 'integer'
784
+ 'description': 'Deprecated. This value is ignored.',
785
+ 'type': 'integer',
786
+ 'hidecmdl': True,
787
+ 'hideconf': True,
783
788
  },
784
789
  'layers:lockmemory': {
785
790
  'default': False,
@@ -857,7 +862,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
857
862
  self.httpextapidb = self.slab.initdb('http:ext:apis')
858
863
 
859
864
  if self.inaugural:
860
- await self.cellinfo.set('cortex:version', s_version.version)
865
+ self.cellinfo.set('cortex:version', s_version.version)
861
866
 
862
867
  corevers = self.cellinfo.get('cortex:version')
863
868
  s_version.reqVersion(corevers, reqver, exc=s_exc.BadStorageVersion,
@@ -881,7 +886,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
881
886
 
882
887
  self.stormmods = {} # name: mdef
883
888
  self.stormpkgs = {} # name: pkgdef
884
- self.stormvars = None # type: s_hive.HiveDict
889
+ self.stormvars = None # type: s_lmdbslab.SafeKeyVal
885
890
 
886
891
  self.svcsbyiden = {}
887
892
  self.svcsbyname = {}
@@ -923,7 +928,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
923
928
  # generic fini handler for the Cortex
924
929
  self.onfini(self._onCoreFini)
925
930
 
926
- await self._initCoreHive()
931
+ self.cortexdata = self.slab.getSafeKeyVal('cortex')
932
+
933
+ await self._initCoreInfo()
927
934
  self._initStormLibs()
928
935
  self._initFeedFuncs()
929
936
 
@@ -943,6 +950,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
943
950
  (1, self._migrateTaxonomyIface),
944
951
  ), nexs=False)
945
952
 
953
+ await self._bumpCellVers('cortex:storage', (
954
+ (1, self._storUpdateMacros),
955
+ (4, self._storCortexHiveMigration),
956
+ ), nexs=False)
957
+
946
958
  # Perform module loading
947
959
  await self._loadCoreMods()
948
960
  await self._loadExtModel()
@@ -961,8 +973,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
961
973
 
962
974
  await self._initOAuthManager()
963
975
 
964
- stormdmonhive = await self.hive.open(('cortex', 'storm', 'dmons'))
965
- self.stormdmonhive = await stormdmonhive.dict()
976
+ self.stormdmondefs = self.cortexdata.getSubKeyVal('storm:dmons:')
966
977
  self.stormdmons = await s_storm.DmonManager.anit(self)
967
978
  self.onfini(self.stormdmons)
968
979
 
@@ -973,15 +984,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
973
984
 
974
985
  await self._initRuntFuncs()
975
986
 
976
- taghive = await self.hive.open(('cortex', 'tagmeta'))
977
- cmdhive = await self.hive.open(('cortex', 'storm', 'cmds'))
978
- pkghive = await self.hive.open(('cortex', 'storm', 'packages'))
979
- svchive = await self.hive.open(('cortex', 'storm', 'services'))
980
-
981
- self.taghive = await taghive.dict()
982
- self.cmdhive = await cmdhive.dict()
983
- self.pkghive = await pkghive.dict()
984
- self.svchive = await svchive.dict()
987
+ self.tagmeta = self.cortexdata.getSubKeyVal('tagmeta:')
988
+ self.cmddefs = self.cortexdata.getSubKeyVal('storm:cmds:')
989
+ self.pkgdefs = self.cortexdata.getSubKeyVal('storm:packages:')
990
+ self.svcdefs = self.cortexdata.getSubKeyVal('storm:services:')
985
991
 
986
992
  await self._initDeprLocks()
987
993
  await self._warnDeprLocks()
@@ -997,29 +1003,87 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
997
1003
 
998
1004
  # TODO - Remove this in 3.0.0
999
1005
  ag = await self.auth.addAuthGate('cortex', 'cortex')
1000
- for (useriden, user) in ag.gateusers.items():
1006
+ for useriden in ag.gateusers.keys():
1007
+ user = self.auth.user(useriden)
1008
+ if user is None:
1009
+ continue
1010
+
1001
1011
  mesg = f'User {useriden} ({user.name}) has a rule on the "cortex" authgate. This authgate is not used ' \
1002
1012
  f'for permission checks and will be removed in Synapse v3.0.0.'
1003
1013
  logger.warning(mesg, extra=await self.getLogExtra(user=useriden, username=user.name))
1004
- for (roleiden, role) in ag.gateroles.items():
1014
+ for roleiden in ag.gateroles.keys():
1015
+ role = self.auth.role(roleiden)
1016
+ if role is None:
1017
+ continue
1018
+
1005
1019
  mesg = f'Role {roleiden} ({role.name}) has a rule on the "cortex" authgate. This authgate is not used ' \
1006
1020
  f'for permission checks and will be removed in Synapse v3.0.0.'
1007
1021
  logger.warning(mesg, extra=await self.getLogExtra(role=roleiden, rolename=role.name))
1008
1022
 
1009
1023
  self._initVaults()
1010
1024
 
1011
- await self._bumpCellVers('cortex:storage', (
1012
- (1, self._storUpdateMacros),
1013
- (2, self._storLayrFeedDefaults),
1014
- (3, self._updateTriggerViewIdens),
1015
- ), nexs=False)
1025
+ async def _storCortexHiveMigration(self):
1026
+
1027
+ logger.warning('migrating Cortex data out of hive')
1028
+
1029
+ viewdefs = self.cortexdata.getSubKeyVal('view:info:')
1030
+ async with await self.hive.open(('cortex', 'views')) as viewnodes:
1031
+ for view_iden, node in viewnodes:
1032
+ viewdict = await node.dict()
1033
+ viewinfo = viewdict.pack()
1034
+ viewinfo.setdefault('iden', view_iden)
1035
+ viewdefs.set(view_iden, viewinfo)
1036
+
1037
+ trigdict = self.cortexdata.getSubKeyVal(f'view:{view_iden}:trigger:')
1038
+ async with await node.open(('triggers',)) as trignodes:
1039
+ for iden, trig in trignodes:
1040
+ valu = trig.valu
1041
+ if valu.get('view', s_common.novalu) != view_iden:
1042
+ valu['view'] = view_iden
1043
+ trigdict.set(iden, valu)
1044
+
1045
+ layrdefs = self.cortexdata.getSubKeyVal('layer:info:')
1046
+ async with await self.hive.open(('cortex', 'layers')) as layrnodes:
1047
+ for iden, node in layrnodes:
1048
+ layrdict = await node.dict()
1049
+ layrinfo = layrdict.pack()
1050
+ pushs = layrinfo.get('pushs', {})
1051
+ if pushs:
1052
+ for pdef in pushs.values():
1053
+ pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1054
+ pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1016
1055
 
1017
- async def _updateTriggerViewIdens(self):
1018
- for view in self.views.values():
1019
- for trigiden, trigger in await view.listTriggers():
1020
- if trigger.get('view') != view.iden:
1021
- trigger.tdef['view'] = view.iden
1022
- await view.trigdict.set(trigiden, trigger.tdef)
1056
+ pulls = layrinfo.get('pulls', {})
1057
+ if pulls:
1058
+ pulls = layrinfo.get('pulls', {})
1059
+ for pdef in pulls.values():
1060
+ pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1061
+ pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1062
+
1063
+ layrdefs.set(iden, layrinfo)
1064
+
1065
+ migrs = (
1066
+ (('agenda', 'appts'), 'agenda:appt:'),
1067
+ (('cortex', 'tagmeta'), 'tagmeta:'),
1068
+ (('cortex', 'storm', 'cmds'), 'storm:cmds:'),
1069
+ (('cortex', 'storm', 'vars'), 'storm:vars:'),
1070
+ (('cortex', 'storm', 'dmons'), 'storm:dmons:'),
1071
+ (('cortex', 'storm', 'packages'), 'storm:packages:'),
1072
+ (('cortex', 'storm', 'services'), 'storm:services:'),
1073
+ (('cortex', 'model', 'forms'), 'model:forms:'),
1074
+ (('cortex', 'model', 'props'), 'model:props:'),
1075
+ (('cortex', 'model', 'univs'), 'model:univs:'),
1076
+ (('cortex', 'model', 'tagprops'), 'model:tagprops:'),
1077
+ (('cortex', 'model', 'deprlocks'), 'model:deprlocks:'),
1078
+ )
1079
+
1080
+ for hivepath, kvpref in migrs:
1081
+ subkv = self.cortexdata.getSubKeyVal(kvpref)
1082
+ async with await self.hive.open(hivepath) as hivenode:
1083
+ for name, node in hivenode:
1084
+ subkv.set(name, node.valu)
1085
+
1086
+ logger.warning('...Cortex data migration complete!')
1023
1087
 
1024
1088
  async def _viewNomergeToProtected(self):
1025
1089
  for view in self.views.values():
@@ -1449,26 +1513,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1449
1513
  role = await self.auth.getRoleByName('all')
1450
1514
  await role.addRule((True, ('layer', 'read')), gateiden=layriden)
1451
1515
 
1452
- async def _storLayrFeedDefaults(self):
1453
-
1454
- for layer in list(self.layers.values()):
1455
- layrinfo = layer.layrinfo # type: s_hive.HiveDict
1456
-
1457
- pushs = layrinfo.get('pushs', {})
1458
- if pushs:
1459
- for pdef in pushs.values():
1460
- pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1461
- pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1462
- await layrinfo.set('pushs', pushs, nexs=False)
1463
-
1464
- pulls = layrinfo.get('pulls', {})
1465
- if pulls:
1466
- pulls = layrinfo.get('pulls', {})
1467
- for pdef in pulls.values():
1468
- pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1469
- pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1470
- await layrinfo.set('pulls', pulls, nexs=False)
1471
-
1472
1516
  async def initServiceRuntime(self):
1473
1517
 
1474
1518
  # do any post-nexus initialization here...
@@ -1598,8 +1642,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1598
1642
  mesg = 'setDeprLock() called on non-existant or non-deprecated form, property, or type.'
1599
1643
  raise s_exc.NoSuchProp(name=name, mesg=mesg)
1600
1644
 
1601
- self.deprlocks[name] = locked
1602
- await self.hive.set(('cortex', 'model', 'deprlocks'), self.deprlocks)
1645
+ self.deprlocks.set(name, locked)
1603
1646
 
1604
1647
  for elem in todo:
1605
1648
  elem.locked = locked
@@ -1905,14 +1948,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1905
1948
  await core.setTagModel("cno.cve", "regex", (None, None, "[0-9]{4}", "[0-9]{5}"))
1906
1949
 
1907
1950
  '''
1908
- meta = self.taghive.get(tagname)
1951
+ meta = self.tagmeta.get(tagname)
1909
1952
  if meta is None:
1910
1953
  meta = {}
1911
1954
 
1912
1955
  meta[name] = valu
1913
1956
  reqValidTagModel(meta)
1914
1957
 
1915
- await self.taghive.set(tagname, meta)
1958
+ self.tagmeta.set(tagname, meta)
1916
1959
 
1917
1960
  # clear cached entries
1918
1961
  if name == 'regex':
@@ -1928,7 +1971,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1928
1971
  Arguments:
1929
1972
  tagname (str): The name of the tag.
1930
1973
  '''
1931
- await self.taghive.pop(tagname)
1974
+ self.tagmeta.pop(tagname)
1932
1975
  self.tagvalid.clear()
1933
1976
  self.tagprune.clear()
1934
1977
 
@@ -1945,12 +1988,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1945
1988
  (object): The current value of the property.
1946
1989
  '''
1947
1990
 
1948
- meta = self.taghive.get(tagname)
1991
+ meta = self.tagmeta.get(tagname)
1949
1992
  if meta is None:
1950
1993
  return None
1951
1994
 
1952
1995
  retn = meta.pop(name, None)
1953
- await self.taghive.set(name, meta)
1996
+ self.tagmeta.set(tagname, meta)
1954
1997
 
1955
1998
  if name == 'regex':
1956
1999
  self.tagvalid.clear()
@@ -1973,7 +2016,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1973
2016
  parts = s_chop.tagpath(tagname)
1974
2017
  for tag in s_chop.tags(tagname):
1975
2018
 
1976
- meta = self.taghive.get(tag)
2019
+ meta = self.tagmeta.get(tag)
1977
2020
  if meta is None:
1978
2021
  continue
1979
2022
 
@@ -2006,7 +2049,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2006
2049
  prune.append(tag)
2007
2050
  continue
2008
2051
 
2009
- meta = self.taghive.get(tag)
2052
+ meta = self.tagmeta.get(tag)
2010
2053
  if meta is None:
2011
2054
  continue
2012
2055
 
@@ -2028,7 +2071,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2028
2071
  Returns:
2029
2072
  (dict): The tag model specification or None.
2030
2073
  '''
2031
- retn = self.taghive.get(tagname)
2074
+ retn = self.tagmeta.get(tagname)
2032
2075
  if retn is not None:
2033
2076
  return dict(retn)
2034
2077
 
@@ -2039,7 +2082,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2039
2082
  Returns:
2040
2083
  ([(str, dict), ...]): A list of tag model specification tuples.
2041
2084
  '''
2042
- return list(self.taghive.items())
2085
+ return list(self.tagmeta.items())
2043
2086
 
2044
2087
  async def _finiStor(self):
2045
2088
  await asyncio.gather(*[view.fini() for view in self.views.values()])
@@ -2085,7 +2128,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2085
2128
 
2086
2129
  async def _initStormDmons(self):
2087
2130
 
2088
- for iden, ddef in self.stormdmonhive.items():
2131
+ for iden, ddef in self.stormdmondefs.items():
2089
2132
  try:
2090
2133
  await self.runStormDmon(iden, ddef)
2091
2134
 
@@ -2097,7 +2140,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2097
2140
 
2098
2141
  async def _initStormSvcs(self):
2099
2142
 
2100
- for iden, sdef in self.svchive.items():
2143
+ for iden, sdef in self.svcdefs.items():
2101
2144
 
2102
2145
  try:
2103
2146
  await self._setStormSvc(sdef)
@@ -2160,7 +2203,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2160
2203
  '''
2161
2204
  name = cdef.get('name')
2162
2205
  await self._setStormCmd(cdef)
2163
- await self.cmdhive.set(name, cdef)
2206
+ self.cmddefs.set(name, cdef)
2164
2207
 
2165
2208
  async def _reqStormCmd(self, cdef):
2166
2209
 
@@ -2496,12 +2539,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2496
2539
  if ctor is None:
2497
2540
  return
2498
2541
 
2499
- cdef = self.cmdhive.get(name)
2542
+ cdef = self.cmddefs.get(name)
2500
2543
  if cdef is None:
2501
2544
  mesg = f'The storm command ({name}) is not dynamic.'
2502
2545
  raise s_exc.CantDelCmd(mesg=mesg)
2503
2546
 
2504
- await self.cmdhive.pop(name)
2547
+ self.cmddefs.pop(name)
2505
2548
  self.stormcmds.pop(name, None)
2506
2549
 
2507
2550
  await self.fire('core:cmd:change', cmd=name, act='del')
@@ -2556,7 +2599,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2556
2599
  @s_nexus.Pusher.onPush('pkg:add')
2557
2600
  async def _addStormPkg(self, pkgdef):
2558
2601
  name = pkgdef.get('name')
2559
- olddef = self.pkghive.get(name, None)
2602
+ olddef = self.pkgdefs.get(name, None)
2560
2603
  if olddef is not None:
2561
2604
  if s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
2562
2605
  await self._dropStormPkg(olddef)
@@ -2564,7 +2607,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2564
2607
  return
2565
2608
 
2566
2609
  await self.loadStormPkg(pkgdef)
2567
- await self.pkghive.set(name, pkgdef)
2610
+ self.pkgdefs.set(name, pkgdef)
2568
2611
 
2569
2612
  self._clearPermDefs()
2570
2613
 
@@ -2577,7 +2620,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2577
2620
  await self.feedBeholder('pkg:add', pkgdef, gates=gates, perms=perms)
2578
2621
 
2579
2622
  async def delStormPkg(self, name):
2580
- pkgdef = self.pkghive.get(name, None)
2623
+ pkgdef = self.pkgdefs.get(name)
2581
2624
  if pkgdef is None:
2582
2625
  mesg = f'No storm package: {name}.'
2583
2626
  raise s_exc.NoSuchPkg(mesg=mesg)
@@ -2589,7 +2632,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2589
2632
  '''
2590
2633
  Delete a storm package by name.
2591
2634
  '''
2592
- pkgdef = await self.pkghive.pop(name, None)
2635
+ pkgdef = self.pkgdefs.pop(name, None)
2593
2636
  if pkgdef is None:
2594
2637
  return
2595
2638
 
@@ -2612,7 +2655,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2612
2655
  return self._getStormPkgs()
2613
2656
 
2614
2657
  def _getStormPkgs(self):
2615
- return copy.deepcopy(list(self.pkghive.values()))
2658
+ return copy.deepcopy(list(self.pkgdefs.values()))
2616
2659
 
2617
2660
  async def getStormMods(self):
2618
2661
  return copy.deepcopy(self.stormmods)
@@ -2927,13 +2970,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2927
2970
  return ssvc.sdef
2928
2971
 
2929
2972
  ssvc = await self._setStormSvc(sdef)
2930
- await self.svchive.set(iden, sdef)
2973
+ self.svcdefs.set(iden, sdef)
2931
2974
 
2932
2975
  await self.feedBeholder('svc:add', {'name': sdef.get('name'), 'iden': iden})
2933
2976
  return ssvc.sdef
2934
2977
 
2935
2978
  async def delStormSvc(self, iden):
2936
- sdef = self.svchive.get(iden)
2979
+ sdef = self.svcdefs.get(iden)
2937
2980
  if sdef is None:
2938
2981
  mesg = f'No storm service with iden: {iden}'
2939
2982
  raise s_exc.NoSuchStormSvc(mesg=mesg, iden=iden)
@@ -2945,7 +2988,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2945
2988
  '''
2946
2989
  Delete a registered storm service from the cortex.
2947
2990
  '''
2948
- sdef = self.svchive.get(iden)
2991
+ sdef = self.svcdefs.get(iden)
2949
2992
  if sdef is None: # pragma: no cover
2950
2993
  return
2951
2994
 
@@ -2957,7 +3000,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2957
3000
  except Exception as e:
2958
3001
  logger.exception(f'service.del hook for service {iden} failed with error: {e}')
2959
3002
 
2960
- sdef = await self.svchive.pop(iden)
3003
+ sdef = self.svcdefs.pop(iden)
2961
3004
 
2962
3005
  await self._delStormSvcPkgs(iden)
2963
3006
 
@@ -2983,7 +3026,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2983
3026
 
2984
3027
  def getStormSvcPkgs(self, iden):
2985
3028
  pkgs = []
2986
- for _, pdef in self.pkghive.items():
3029
+ for _, pdef in self.pkgdefs.items():
2987
3030
  pkgiden = pdef.get('svciden')
2988
3031
  if pkgiden and pkgiden == iden:
2989
3032
  pkgs.append(pdef)
@@ -3020,17 +3063,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3020
3063
  Returns:
3021
3064
  dict: An updated storm service definition dictionary.
3022
3065
  '''
3023
- sdef = self.svchive.get(iden)
3066
+ sdef = self.svcdefs.get(iden)
3024
3067
  if sdef is None:
3025
3068
  mesg = f'No storm service with iden: {iden}'
3026
3069
  raise s_exc.NoSuchStormSvc(mesg=mesg)
3027
3070
 
3028
3071
  sdef['evts'] = edef
3029
- await self.svchive.set(iden, sdef)
3072
+ self.svcdefs.set(iden, sdef)
3030
3073
  return sdef
3031
3074
 
3032
3075
  async def _runStormSvcAdd(self, iden):
3033
- sdef = self.svchive.get(iden)
3076
+ sdef = self.svcdefs.get(iden)
3034
3077
  if sdef is None:
3035
3078
  mesg = f'No storm service with iden: {iden}'
3036
3079
  raise s_exc.NoSuchStormSvc(mesg=mesg)
@@ -3047,12 +3090,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3047
3090
  return
3048
3091
 
3049
3092
  sdef['added'] = True
3050
- await self.svchive.set(iden, sdef)
3093
+ self.svcdefs.set(iden, sdef)
3051
3094
 
3052
3095
  async def runStormSvcEvent(self, iden, name):
3053
3096
  assert name in ('add', 'del')
3054
3097
 
3055
- sdef = self.svchive.get(iden)
3098
+ sdef = self.svcdefs.get(iden)
3056
3099
  if sdef is None:
3057
3100
  mesg = f'No storm service with iden: {iden}'
3058
3101
  raise s_exc.NoSuchStormSvc(mesg=mesg)
@@ -3085,15 +3128,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3085
3128
  # Global stormvars APIs
3086
3129
 
3087
3130
  async def getStormVar(self, name, default=None):
3088
- return self.stormvars.get(name, default=default)
3131
+ return self.stormvars.get(name, defv=default)
3089
3132
 
3090
3133
  @s_nexus.Pusher.onPushAuto('stormvar:pop')
3091
3134
  async def popStormVar(self, name, default=None):
3092
- return await self.stormvars.pop(name, default=default)
3135
+ return self.stormvars.pop(name, defv=default)
3093
3136
 
3094
3137
  @s_nexus.Pusher.onPushAuto('stormvar:set')
3095
3138
  async def setStormVar(self, name, valu):
3096
- return await self.stormvars.set(name, valu)
3139
+ return self.stormvars.set(name, valu)
3097
3140
 
3098
3141
  async def itemsStormVar(self):
3099
3142
  for item in self.stormvars.items():
@@ -3125,17 +3168,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3125
3168
 
3126
3169
  async def _loadExtModel(self):
3127
3170
 
3128
- self.extforms = await (await self.hive.open(('cortex', 'model', 'forms'))).dict()
3129
- self.extprops = await (await self.hive.open(('cortex', 'model', 'props'))).dict()
3130
- self.extunivs = await (await self.hive.open(('cortex', 'model', 'univs'))).dict()
3131
- self.exttagprops = await (await self.hive.open(('cortex', 'model', 'tagprops'))).dict()
3171
+ self.extforms = self.cortexdata.getSubKeyVal('model:forms:')
3172
+ self.extprops = self.cortexdata.getSubKeyVal('model:props:')
3173
+ self.extunivs = self.cortexdata.getSubKeyVal('model:univs:')
3174
+ self.extedges = self.cortexdata.getSubKeyVal('model:edges:')
3175
+ self.exttagprops = self.cortexdata.getSubKeyVal('model:tagprops:')
3132
3176
 
3133
3177
  for formname, basetype, typeopts, typeinfo in self.extforms.values():
3134
3178
  try:
3135
3179
  self.model.addType(formname, basetype, typeopts, typeinfo)
3136
3180
  form = self.model.addForm(formname, {}, ())
3137
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3138
- raise
3139
3181
  except Exception as e:
3140
3182
  logger.warning(f'Extended form ({formname}) error: {e}')
3141
3183
  else:
@@ -3147,8 +3189,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3147
3189
  for form, prop, tdef, info in self.extprops.values():
3148
3190
  try:
3149
3191
  prop = self.model.addFormProp(form, prop, tdef, info)
3150
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3151
- raise
3152
3192
  except Exception as e:
3153
3193
  logger.warning(f'ext prop ({form}:{prop}) error: {e}')
3154
3194
  else:
@@ -3160,19 +3200,21 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3160
3200
  for prop, tdef, info in self.extunivs.values():
3161
3201
  try:
3162
3202
  self.model.addUnivProp(prop, tdef, info)
3163
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3164
- raise
3165
3203
  except Exception as e:
3166
3204
  logger.warning(f'ext univ ({prop}) error: {e}')
3167
3205
 
3168
3206
  for prop, tdef, info in self.exttagprops.values():
3169
3207
  try:
3170
3208
  self.model.addTagProp(prop, tdef, info)
3171
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3172
- raise
3173
3209
  except Exception as e:
3174
3210
  logger.warning(f'ext tag prop ({prop}) error: {e}')
3175
3211
 
3212
+ for edge, info in self.extedges.values():
3213
+ try:
3214
+ self.model.addEdge(edge, info)
3215
+ except Exception as e:
3216
+ logger.warning(f'ext edge ({edge}) error: {e}')
3217
+
3176
3218
  async def getExtModel(self):
3177
3219
  '''
3178
3220
  Get all extended model properties in the Cortex.
@@ -3192,6 +3234,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3192
3234
 
3193
3235
  for prop, tdef, info in self.exttagprops.values():
3194
3236
  ret['tagprops'].append((prop, tdef, info))
3237
+
3238
+ for edge, info in self.extedges.values():
3239
+ ret['edges'].append((edge, info))
3240
+
3195
3241
  ret['version'] = (1, 0)
3196
3242
  return copy.deepcopy(dict(ret))
3197
3243
 
@@ -3206,9 +3252,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3206
3252
  Bool: True when the model was added.
3207
3253
 
3208
3254
  Raises:
3209
- s_exc.BadFormDef: If a form exists with a different definition the provided definition.
3210
- s_exc.BadPropDef: If a propery, tagprop, or universal propert from exists with a different definition
3255
+ s_exc.BadFormDef: If a form exists with a different definition than the provided definition.
3256
+ s_exc.BadPropDef: If a property, tagprop, or universal property exists with a different definition
3211
3257
  than the provided definition.
3258
+ s_exc.BadEdgeDef: If an edge exists with a different definition than the provided definition.
3212
3259
  '''
3213
3260
 
3214
3261
  # Get our current model definition
@@ -3219,11 +3266,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3219
3266
  props = {(info[0], info[1]): info for info in model.get('props', ())}
3220
3267
  tagprops = {info[0]: info for info in model.get('tagprops', ())}
3221
3268
  univs = {info[0]: info for info in model.get('univs', ())}
3269
+ edges = {info[0]: info for info in model.get('edges', ())}
3222
3270
 
3223
3271
  efrms = {info[0]: info for info in emodl.get('forms', ())}
3224
3272
  eprops = {(info[0], info[1]): info for info in emodl.get('props', ())}
3225
3273
  etagprops = {info[0]: info for info in emodl.get('tagprops', ())}
3226
3274
  eunivs = {info[0]: info for info in emodl.get('univs', ())}
3275
+ eedges = {info[0]: info for info in emodl.get('edges', ())}
3227
3276
 
3228
3277
  for (name, info) in forms.items():
3229
3278
  enfo = efrms.get(name)
@@ -3233,7 +3282,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3233
3282
  if enfo == info:
3234
3283
  continue
3235
3284
  mesg = f'Extended form definition differs from existing definition for {name}.'
3236
- raise s_exc.BadFormDef(mesg)
3285
+ raise s_exc.BadFormDef(mesg=mesg, name=name)
3237
3286
 
3238
3287
  for (name, info) in props.items():
3239
3288
  enfo = eprops.get(name)
@@ -3243,7 +3292,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3243
3292
  if enfo == info:
3244
3293
  continue
3245
3294
  mesg = f'Extended prop definition differs from existing definition for {name}'
3246
- raise s_exc.BadPropDef(mesg)
3295
+ raise s_exc.BadPropDef(mesg=mesg, name=name)
3247
3296
 
3248
3297
  for (name, info) in tagprops.items():
3249
3298
  enfo = etagprops.get(name)
@@ -3253,7 +3302,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3253
3302
  if enfo == info:
3254
3303
  continue
3255
3304
  mesg = f'Extended tagprop definition differs from existing definition for {name}'
3256
- raise s_exc.BadPropDef(mesg)
3305
+ raise s_exc.BadPropDef(mesg=mesg, name=name)
3257
3306
 
3258
3307
  for (name, info) in univs.items():
3259
3308
  enfo = eunivs.get(name)
@@ -3262,8 +3311,20 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3262
3311
  continue
3263
3312
  if enfo == info:
3264
3313
  continue
3265
- mesg = f'Extended universal poroperty definition differs from existing definition for {name}'
3266
- raise s_exc.BadPropDef(mesg)
3314
+ mesg = f'Extended universal property definition differs from existing definition for {name}'
3315
+ raise s_exc.BadPropDef(mesg=mesg, name=name)
3316
+
3317
+ for (name, info) in edges.items():
3318
+ enfo = eedges.get(name)
3319
+ if enfo is None:
3320
+ amodl['edges'].append(info)
3321
+ continue
3322
+ if enfo == info:
3323
+ continue
3324
+
3325
+ (n1form, verb, n2form) = info[0]
3326
+ mesg = f'Extended edge definition differs from existing definition for {info[0]}'
3327
+ raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
3267
3328
 
3268
3329
  for formname, basetype, typeopts, typeinfo in amodl['forms']:
3269
3330
  await self.addForm(formname, basetype, typeopts, typeinfo)
@@ -3277,6 +3338,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3277
3338
  for prop, tdef, info in amodl['univs']:
3278
3339
  await self.addUnivProp(prop, tdef, info)
3279
3340
 
3341
+ for edge, info in amodl['edges']:
3342
+ await self.addEdge(edge, info)
3343
+
3280
3344
  return True
3281
3345
 
3282
3346
  async def addUnivProp(self, name, tdef, info):
@@ -3307,7 +3371,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3307
3371
 
3308
3372
  self.model.addUnivProp(name, tdef, info)
3309
3373
 
3310
- await self.extunivs.set(name, (name, tdef, info))
3374
+ self.extunivs.set(name, (name, tdef, info))
3311
3375
  await self.fire('core:extmodel:change', prop=name, act='add', type='univ')
3312
3376
  base = '.' + name
3313
3377
  univ = self.model.univ(base)
@@ -3349,7 +3413,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3349
3413
  self.model.addType(formname, basetype, typeopts, typeinfo)
3350
3414
  self.model.addForm(formname, {}, ())
3351
3415
 
3352
- await self.extforms.set(formname, (formname, basetype, typeopts, typeinfo))
3416
+ self.extforms.set(formname, (formname, basetype, typeopts, typeinfo))
3353
3417
  await self.fire('core:extmodel:change', form=formname, act='add', type='form')
3354
3418
  form = self.model.form(formname)
3355
3419
  ftyp = self.model.type(formname)
@@ -3379,7 +3443,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3379
3443
  self.model.delForm(formname)
3380
3444
  self.model.delType(formname)
3381
3445
 
3382
- await self.extforms.pop(formname, None)
3446
+ self.extforms.pop(formname, None)
3383
3447
  await self.fire('core:extmodel:change', form=formname, act='del', type='form')
3384
3448
  await self.feedBeholder('model:form:del', {'form': formname})
3385
3449
 
@@ -3415,7 +3479,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3415
3479
  logger.warning(mesg)
3416
3480
 
3417
3481
  full = f'{form}:{prop}'
3418
- await self.extprops.set(full, (form, prop, tdef, info))
3482
+ self.extprops.set(full, (form, prop, tdef, info))
3419
3483
  await self.fire('core:extmodel:change', form=form, prop=prop, act='add', type='formprop')
3420
3484
  prop = self.model.prop(full)
3421
3485
  if prop:
@@ -3448,7 +3512,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3448
3512
  raise s_exc.CantDelProp(mesg=mesg)
3449
3513
 
3450
3514
  self.model.delFormProp(form, prop)
3451
- await self.extprops.pop(full, None)
3515
+ self.extprops.pop(full, None)
3452
3516
  await self.fire('core:extmodel:change',
3453
3517
  form=form, prop=prop, act='del', type='formprop')
3454
3518
 
@@ -3478,7 +3542,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3478
3542
  raise s_exc.CantDelUniv(mesg=mesg)
3479
3543
 
3480
3544
  self.model.delUnivProp(prop)
3481
- await self.extunivs.pop(prop, None)
3545
+ self.extunivs.pop(prop, None)
3482
3546
  await self.fire('core:extmodel:change', name=prop, act='del', type='univ')
3483
3547
  await self.feedBeholder('model:univ:del', {'prop': univname})
3484
3548
 
@@ -3503,7 +3567,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3503
3567
 
3504
3568
  self.model.addTagProp(name, tdef, info)
3505
3569
 
3506
- await self.exttagprops.set(name, (name, tdef, info))
3570
+ self.exttagprops.set(name, (name, tdef, info))
3507
3571
  await self.fire('core:tagprop:change', name=name, act='add')
3508
3572
  tagp = self.model.tagprop(name)
3509
3573
  if tagp:
@@ -3530,10 +3594,60 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3530
3594
 
3531
3595
  self.model.delTagProp(name)
3532
3596
 
3533
- await self.exttagprops.pop(name, None)
3597
+ self.exttagprops.pop(name, None)
3534
3598
  await self.fire('core:tagprop:change', name=name, act='del')
3535
3599
  await self.feedBeholder('model:tagprop:del', {'tagprop': name})
3536
3600
 
3601
+ async def addEdge(self, edge, edgeinfo):
3602
+ if not isinstance(edgeinfo, dict):
3603
+ mesg = 'Edge info should be a dict.'
3604
+ raise s_exc.BadArg(mesg=mesg, edgeinfo=edgeinfo)
3605
+
3606
+ (n1form, verb, n2form) = edge
3607
+ if not verb.startswith('_'):
3608
+ mesg = f'Extended edge verb must begin with "_"; got {verb}'
3609
+ raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
3610
+
3611
+ if n1form is not None:
3612
+ self.model._reqFormName(n1form)
3613
+
3614
+ if n2form is not None:
3615
+ self.model._reqFormName(n2form)
3616
+
3617
+ if self.model.edge(edge) is not None:
3618
+ raise s_exc.DupEdgeType.init(edge)
3619
+
3620
+ return await self._push('model:edge:add', edge, edgeinfo)
3621
+
3622
+ @s_nexus.Pusher.onPush('model:edge:add')
3623
+ async def _addEdge(self, edge, edgeinfo):
3624
+ if self.model.edge(edge) is not None:
3625
+ return
3626
+
3627
+ self.model.addEdge(edge, edgeinfo)
3628
+
3629
+ self.extedges.set(s_common.guid(edge), (edge, edgeinfo))
3630
+ await self.fire('core:extmodel:change', edge=edge, act='add', type='edge')
3631
+ await self.feedBeholder('model:edge:add', {'edge': edge, 'info': edgeinfo})
3632
+
3633
+ async def delEdge(self, edge):
3634
+ if self.extedges.get(s_common.guid(edge)) is None:
3635
+ raise s_exc.NoSuchEdge.init(edge)
3636
+
3637
+ return await self._push('model:edge:del', edge)
3638
+
3639
+ @s_nexus.Pusher.onPush('model:edge:del')
3640
+ async def _delEdge(self, edge):
3641
+ edgeguid = s_common.guid(edge)
3642
+ if self.extedges.get(edgeguid) is None:
3643
+ return
3644
+
3645
+ self.model.delEdge(edge)
3646
+
3647
+ self.extedges.pop(edgeguid, None)
3648
+ await self.fire('core:extmodel:change', edge=edge, act='del', type='edge')
3649
+ await self.feedBeholder('model:edge:del', {'edge': edge})
3650
+
3537
3651
  async def addNodeTag(self, user, iden, tag, valu=(None, None)):
3538
3652
  '''
3539
3653
  Add a tag to a node specified by iden.
@@ -3821,15 +3935,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3821
3935
 
3822
3936
  await self.waitfini(1)
3823
3937
 
3824
- async def _initCoreHive(self):
3825
- stormvarsnode = await self.hive.open(('cortex', 'storm', 'vars'))
3826
- self.stormvars = await stormvarsnode.dict()
3938
+ async def _initCoreInfo(self):
3939
+ self.stormvars = self.cortexdata.getSubKeyVal('storm:vars:')
3827
3940
  if self.inaugural:
3828
- await self.stormvars.set(s_stormlib_cell.runtime_fixes_key, s_stormlib_cell.getMaxHotFixes())
3829
- self.onfini(self.stormvars)
3941
+ self.stormvars.set(s_stormlib_cell.runtime_fixes_key, s_stormlib_cell.getMaxHotFixes())
3830
3942
 
3831
3943
  async def _initDeprLocks(self):
3832
- self.deprlocks = await self.hive.get(('cortex', 'model', 'deprlocks'), {}) # type: s_hive.Node
3944
+ self.deprlocks = self.cortexdata.getSubKeyVal('model:deprlocks:')
3833
3945
  # TODO: 3.0.0 conversion will truncate this hive key
3834
3946
 
3835
3947
  if self.inaugural:
@@ -4062,18 +4174,18 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4062
4174
 
4063
4175
  async def _initPureStormCmds(self):
4064
4176
  oldcmds = []
4065
- for name, cdef in self.cmdhive.items():
4177
+ for name, cdef in self.cmddefs.items():
4066
4178
  cmdiden = cdef.get('cmdconf', {}).get('svciden')
4067
- if cmdiden and self.svchive.get(cmdiden) is None:
4179
+ if cmdiden and self.svcdefs.get(cmdiden) is None:
4068
4180
  oldcmds.append(name)
4069
4181
  else:
4070
4182
  await self._trySetStormCmd(name, cdef)
4071
4183
 
4072
4184
  for name in oldcmds:
4073
4185
  logger.warning(f'Removing old command: [{name}]')
4074
- await self.cmdhive.pop(name)
4186
+ self.cmddefs.pop(name)
4075
4187
 
4076
- for pkgdef in self.pkghive.values():
4188
+ for pkgdef in self.pkgdefs.values():
4077
4189
  await self._tryLoadStormPkg(pkgdef)
4078
4190
 
4079
4191
  async def _trySetStormCmd(self, name, cdef):
@@ -4399,9 +4511,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4399
4511
  mrev = s_modelrev.ModelRev(self)
4400
4512
  await mrev.revCoreLayers()
4401
4513
 
4402
- async def _loadView(self, node):
4514
+ async def _loadView(self, vdef):
4403
4515
 
4404
- view = await self.viewctor(self, node)
4516
+ view = await self.viewctor(self, vdef)
4405
4517
 
4406
4518
  self.views[view.iden] = view
4407
4519
  self.dynitems[view.iden] = view
@@ -4425,8 +4537,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4425
4537
 
4426
4538
  defiden = self.cellinfo.get('defaultview')
4427
4539
 
4428
- for iden, node in await self.hive.open(('cortex', 'views')):
4429
- view = await self._loadView(node)
4540
+ self.viewdefs = self.cortexdata.getSubKeyVal('view:info:')
4541
+
4542
+ for iden, vdef in self.viewdefs.items():
4543
+ view = await self._loadView(vdef)
4430
4544
  if iden == defiden:
4431
4545
  self.view = view
4432
4546
 
@@ -4450,7 +4564,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4450
4564
  }
4451
4565
  vdef = await self.addView(vdef, nexs=False)
4452
4566
  iden = vdef.get('iden')
4453
- await self.cellinfo.set('defaultview', iden)
4567
+ self.cellinfo.set('defaultview', iden)
4454
4568
  self.view = self.getView(iden)
4455
4569
 
4456
4570
  self._calcViewsByLayer()
@@ -4497,13 +4611,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4497
4611
  role = await self.auth.getRoleByName('all')
4498
4612
  await role.addRule((True, ('view', 'read')), gateiden=iden, nexs=False)
4499
4613
 
4500
- node = await self.hive.open(('cortex', 'views', iden))
4614
+ self.viewdefs.set(iden, vdef)
4501
4615
 
4502
- info = await node.dict()
4503
- for name, valu in vdef.items():
4504
- await info.set(name, valu)
4505
-
4506
- view = await self._loadView(node)
4616
+ view = await self._loadView(vdef)
4507
4617
  view.init2()
4508
4618
 
4509
4619
  self._calcViewsByLayer()
@@ -4546,7 +4656,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4546
4656
 
4547
4657
  if (view := self.views.get(viewiden)) is not None:
4548
4658
 
4549
- await self.hive.pop(('cortex', 'views', viewiden))
4659
+ self.viewdefs.pop(viewiden)
4550
4660
  await view.delete()
4551
4661
 
4552
4662
  self._calcViewsByLayer()
@@ -4563,8 +4673,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4563
4673
  continue
4564
4674
 
4565
4675
  view.layers = [lyr for lyr in view.layers if lyr.iden != layriden]
4676
+
4566
4677
  layridens = [lyr.iden for lyr in view.layers]
4567
- await view.info.set('layers', layridens)
4678
+ view.info['layers'] = layridens
4568
4679
 
4569
4680
  mesg = {'iden': view.iden, 'layers': layridens}
4570
4681
  await self.feedBeholder('view:setlayers', mesg, gates=[view.iden, layridens[0]])
@@ -4572,14 +4683,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4572
4683
  if view.parent.iden == viewiden:
4573
4684
  if newparent is None:
4574
4685
  view.parent = None
4575
- await view.info.pop('parent')
4686
+ view.info['parent'] = None
4576
4687
  else:
4577
4688
  view.parent = newview
4578
- await view.info.set('parent', newparent)
4689
+ view.info['parent'] = newparent
4579
4690
 
4580
4691
  mesg = {'iden': view.iden, 'name': 'parent', 'valu': newparent}
4581
4692
  await self.feedBeholder('view:set', mesg, gates=[view.iden, layridens[0]])
4582
4693
 
4694
+ self.viewdefs.set(view.iden, view.info)
4695
+
4583
4696
  if not layrinuse and (layr := self.layers.get(layriden)) is not None:
4584
4697
  del self.layers[layriden]
4585
4698
 
@@ -4593,8 +4706,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4593
4706
  await self.auth.delAuthGate(layriden)
4594
4707
  self.dynitems.pop(layriden)
4595
4708
 
4596
- await self.hive.pop(('cortex', 'layers', layriden))
4597
-
4709
+ self.layerdefs.pop(layriden)
4598
4710
  await layr.delete()
4599
4711
 
4600
4712
  layr.deloffs = nexsitem[0]
@@ -4629,7 +4741,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4629
4741
  if cview.parent is not None and cview.parent.iden == iden:
4630
4742
  raise s_exc.SynErr(mesg='Cannot delete a view that has children')
4631
4743
 
4632
- await self.hive.pop(('cortex', 'views', iden))
4744
+ self.viewdefs.pop(iden)
4633
4745
  await view.delete()
4634
4746
 
4635
4747
  self._calcViewsByLayer()
@@ -4665,7 +4777,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4665
4777
  await self.auth.delAuthGate(iden)
4666
4778
  self.dynitems.pop(iden)
4667
4779
 
4668
- await self.hive.pop(('cortex', 'layers', iden))
4780
+ self.layerdefs.pop(iden)
4669
4781
 
4670
4782
  await layr.delete()
4671
4783
 
@@ -4811,6 +4923,60 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4811
4923
  else:
4812
4924
  return await self._addLayer(ldef, (None, None))
4813
4925
 
4926
+ async def _twinLayer(self, oldlayr):
4927
+
4928
+ newldef = s_msgpack.deepcopy(oldlayr.layrinfo)
4929
+
4930
+ newldef.pop('iden', None)
4931
+
4932
+ newldef = await self.addLayer(newldef)
4933
+ newlayr = self.reqLayer(newldef.get('iden'))
4934
+
4935
+ oldinfo = self.auth.reqAuthGate(oldlayr.iden).pack()
4936
+
4937
+ for userinfo in oldinfo.get('users', ()):
4938
+
4939
+ user = self.auth.user(userinfo.get('iden'))
4940
+ if user is None: # pragma: no cover
4941
+ continue
4942
+
4943
+ if userinfo.get('admin'):
4944
+ await user.setAdmin(True, gateiden=newlayr.iden)
4945
+
4946
+ for rule in userinfo.get('rules', ()):
4947
+ await user.addRule(rule, gateiden=newlayr.iden)
4948
+
4949
+ for roleinfo in oldinfo.get('roles', ()):
4950
+
4951
+ role = self.auth.role(roleinfo.get('iden'))
4952
+ if role is None: # pragma: no cover
4953
+ continue
4954
+
4955
+ for rule in roleinfo.get('rules', ()):
4956
+ await role.addRule(rule, gateiden=newlayr.iden)
4957
+
4958
+ return newlayr
4959
+
4960
+ @s_nexus.Pusher.onPushAuto('layer:swap')
4961
+ async def swapLayer(self, oldiden, newiden):
4962
+ '''
4963
+ Atomically swap out a layer from all views that contain it.
4964
+ '''
4965
+ self.reqLayer(oldiden)
4966
+ self.reqLayer(newiden)
4967
+
4968
+ for view in list(self.views.values()):
4969
+ await asyncio.sleep(0)
4970
+
4971
+ oldlayers = view.info.get('layers')
4972
+ if oldiden not in oldlayers:
4973
+ continue
4974
+
4975
+ newlayers = list(oldlayers)
4976
+ newlayers[oldlayers.index(oldiden)] = newiden
4977
+
4978
+ await view._setLayerIdens(newlayers)
4979
+
4814
4980
  @s_nexus.Pusher.onPush('layer:add', passitem=True)
4815
4981
  async def _addLayer(self, ldef, nexsitem):
4816
4982
 
@@ -4827,13 +4993,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4827
4993
 
4828
4994
  user = await self.auth.reqUser(creator)
4829
4995
 
4830
- node = await self.hive.open(('cortex', 'layers', iden))
4831
-
4832
- layrinfo = await node.dict()
4833
- for name, valu in ldef.items():
4834
- await layrinfo.set(name, valu)
4996
+ self.layerdefs.set(iden, ldef)
4835
4997
 
4836
- layr = await self._initLayr(layrinfo, nexsoffs=nexsitem[0])
4998
+ layr = await self._initLayr(ldef, nexsoffs=nexsitem[0])
4837
4999
  await user.setAdmin(True, gateiden=iden, logged=False)
4838
5000
 
4839
5001
  # forward wind the new layer to the current model version
@@ -4899,10 +5061,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4899
5061
  return await s_layer.Layer.anit(self, layrinfo)
4900
5062
 
4901
5063
  async def _initCoreLayers(self):
4902
- node = await self.hive.open(('cortex', 'layers'))
4903
- for _, node in node:
4904
- layrinfo = await node.dict()
4905
- await self._initLayr(layrinfo)
5064
+ self.layerdefs = self.cortexdata.getSubKeyVal('layer:info:')
5065
+ for ldef in self.layerdefs.values():
5066
+ await self._initLayr(ldef)
4906
5067
 
4907
5068
  @s_nexus.Pusher.onPushAuto('layer:push:add')
4908
5069
  async def addLayrPush(self, layriden, pdef):
@@ -4925,7 +5086,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4925
5086
 
4926
5087
  pushs[iden] = pdef
4927
5088
 
4928
- await layr.layrinfo.set('pushs', pushs)
5089
+ layr.layrinfo['pushs'] = pushs
5090
+ self.layerdefs.set(layr.iden, layr.layrinfo)
5091
+
4929
5092
  await self.runLayrPush(layr, pdef)
4930
5093
 
4931
5094
  @s_nexus.Pusher.onPushAuto('layer:push:del')
@@ -4943,7 +5106,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4943
5106
  if pdef is None:
4944
5107
  return
4945
5108
 
4946
- await layr.layrinfo.set('pushs', pushs)
5109
+ layr.layrinfo['pushs'] = pushs
5110
+ self.layerdefs.set(layr.iden, layr.layrinfo)
5111
+
4947
5112
  await self.delActiveCoro(pushiden)
4948
5113
 
4949
5114
  @s_nexus.Pusher.onPushAuto('layer:pull:add')
@@ -4966,7 +5131,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4966
5131
  return
4967
5132
 
4968
5133
  pulls[iden] = pdef
4969
- await layr.layrinfo.set('pulls', pulls)
5134
+
5135
+ layr.layrinfo['pulls'] = pulls
5136
+ self.layerdefs.set(layr.iden, layr.layrinfo)
4970
5137
 
4971
5138
  await self.runLayrPull(layr, pdef)
4972
5139
 
@@ -4985,7 +5152,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4985
5152
  if pdef is None:
4986
5153
  return
4987
5154
 
4988
- await layr.layrinfo.set('pulls', pulls)
5155
+ layr.layrinfo['pulls'] = pulls
5156
+ self.layerdefs.set(layr.iden, layr.layrinfo)
5157
+
4989
5158
  await self.delActiveCoro(pulliden)
4990
5159
 
4991
5160
  async def runLayrPush(self, layr, pdef):
@@ -5106,24 +5275,20 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5106
5275
  newpath = s_common.gendir(self.dirn, 'layers', newiden)
5107
5276
  await layr.clone(newpath)
5108
5277
 
5109
- node = await self.hive.open(('cortex', 'layers', iden))
5110
- copynode = await self.hive.open(('cortex', 'layers', newiden))
5278
+ copyinfo = self.layerdefs.get(iden)
5111
5279
 
5112
- layrinfo = await node.dict()
5113
- copyinfo = await copynode.dict()
5114
- for name, valu in layrinfo.items():
5115
- await copyinfo.set(name, valu)
5280
+ for name, valu in copyinfo.items():
5281
+ ldef.setdefault(name, valu)
5116
5282
 
5117
- for name, valu in ldef.items():
5118
- await copyinfo.set(name, valu)
5283
+ self.layerdefs.set(newiden, ldef)
5119
5284
 
5120
- copylayr = await self._initLayr(copyinfo, nexsoffs=nexsitem[0])
5285
+ copylayr = await self._initLayr(ldef, nexsoffs=nexsitem[0])
5121
5286
 
5122
5287
  creator = copyinfo.get('creator')
5123
5288
  user = await self.auth.reqUser(creator)
5124
5289
  await user.setAdmin(True, gateiden=newiden, logged=False)
5125
5290
 
5126
- return await copylayr.pack()
5291
+ return ldef
5127
5292
 
5128
5293
  def addStormCmd(self, ctor):
5129
5294
  '''
@@ -5149,7 +5314,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5149
5314
 
5150
5315
  @s_nexus.Pusher.onPushAuto('storm:dmon:bump')
5151
5316
  async def bumpStormDmon(self, iden):
5152
- ddef = self.stormdmonhive.get(iden)
5317
+ ddef = self.stormdmondefs.get(iden)
5153
5318
  if ddef is None:
5154
5319
  return False
5155
5320
 
@@ -5166,7 +5331,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5166
5331
  Args:
5167
5332
  iden (str): User iden.
5168
5333
  '''
5169
- for dmoniden, ddef in list(self.stormdmonhive.items()):
5334
+ for dmoniden, ddef in list(self.stormdmondefs.items()):
5170
5335
  if ddef.get('user') == iden:
5171
5336
  await self.bumpStormDmon(dmoniden)
5172
5337
 
@@ -5182,7 +5347,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5182
5347
  dmon.enabled = True
5183
5348
  dmon.ddef['enabled'] = True
5184
5349
 
5185
- await self.stormdmonhive.set(iden, dmon.ddef)
5350
+ self.stormdmondefs.set(iden, dmon.ddef)
5186
5351
 
5187
5352
  if self.isactive:
5188
5353
  await dmon.run()
@@ -5202,7 +5367,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5202
5367
  dmon.enabled = False
5203
5368
  dmon.ddef['enabled'] = False
5204
5369
 
5205
- await self.stormdmonhive.set(iden, dmon.ddef)
5370
+ self.stormdmondefs.set(iden, dmon.ddef)
5206
5371
 
5207
5372
  if self.isactive:
5208
5373
  await dmon.stop()
@@ -5223,14 +5388,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5223
5388
 
5224
5389
  dmon = await self.runStormDmon(iden, ddef)
5225
5390
 
5226
- await self.stormdmonhive.set(iden, ddef)
5391
+ self.stormdmondefs.set(iden, ddef)
5227
5392
  return dmon.pack()
5228
5393
 
5229
5394
  async def delStormDmon(self, iden):
5230
5395
  '''
5231
5396
  Stop and remove a storm dmon.
5232
5397
  '''
5233
- ddef = self.stormdmonhive.get(iden)
5398
+ ddef = self.stormdmondefs.get(iden)
5234
5399
  if ddef is None:
5235
5400
  mesg = f'No storm daemon exists with iden {iden}.'
5236
5401
  raise s_exc.NoSuchIden(mesg=mesg)
@@ -5239,7 +5404,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5239
5404
 
5240
5405
  @s_nexus.Pusher.onPush('storm:dmon:del')
5241
5406
  async def _delStormDmon(self, iden):
5242
- ddef = await self.stormdmonhive.pop(iden)
5407
+ ddef = self.stormdmondefs.pop(iden)
5243
5408
  if ddef is None: # pragma: no cover
5244
5409
  return
5245
5410
  await self.stormdmons.popDmon(iden)