synapse 2.176.0__py311-none-any.whl → 2.177.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (70) hide show
  1. synapse/axon.py +24 -9
  2. synapse/cortex.py +329 -168
  3. synapse/cryotank.py +46 -37
  4. synapse/datamodel.py +17 -4
  5. synapse/exc.py +19 -0
  6. synapse/lib/agenda.py +7 -13
  7. synapse/lib/auth.py +1520 -0
  8. synapse/lib/cell.py +255 -53
  9. synapse/lib/grammar.py +5 -0
  10. synapse/lib/hive.py +24 -3
  11. synapse/lib/hiveauth.py +6 -32
  12. synapse/lib/layer.py +7 -4
  13. synapse/lib/link.py +21 -17
  14. synapse/lib/lmdbslab.py +149 -0
  15. synapse/lib/modelrev.py +1 -1
  16. synapse/lib/schemas.py +136 -0
  17. synapse/lib/storm.py +61 -29
  18. synapse/lib/stormlib/aha.py +1 -1
  19. synapse/lib/stormlib/auth.py +185 -10
  20. synapse/lib/stormlib/cortex.py +16 -5
  21. synapse/lib/stormlib/gen.py +80 -0
  22. synapse/lib/stormlib/model.py +55 -0
  23. synapse/lib/stormlib/modelext.py +60 -0
  24. synapse/lib/stormlib/tabular.py +212 -0
  25. synapse/lib/stormtypes.py +14 -1
  26. synapse/lib/trigger.py +1 -1
  27. synapse/lib/version.py +2 -2
  28. synapse/lib/view.py +55 -28
  29. synapse/models/base.py +7 -0
  30. synapse/models/biz.py +4 -0
  31. synapse/models/files.py +8 -1
  32. synapse/models/inet.py +8 -0
  33. synapse/tests/files/changelog/model_2.176.0_16ee721a6b7221344eaf946c3ab4602dda546b1a.yaml.gz +0 -0
  34. synapse/tests/files/changelog/model_2.176.0_2a25c58bbd344716cd7cbc3f4304d8925b0f4ef2.yaml.gz +0 -0
  35. synapse/tests/test_axon.py +7 -4
  36. synapse/tests/test_cortex.py +127 -82
  37. synapse/tests/test_cryotank.py +4 -4
  38. synapse/tests/test_datamodel.py +7 -0
  39. synapse/tests/test_lib_agenda.py +7 -0
  40. synapse/tests/{test_lib_hiveauth.py → test_lib_auth.py} +314 -11
  41. synapse/tests/test_lib_cell.py +161 -8
  42. synapse/tests/test_lib_httpapi.py +18 -14
  43. synapse/tests/test_lib_layer.py +33 -33
  44. synapse/tests/test_lib_link.py +42 -1
  45. synapse/tests/test_lib_lmdbslab.py +68 -0
  46. synapse/tests/test_lib_nexus.py +4 -4
  47. synapse/tests/test_lib_node.py +0 -7
  48. synapse/tests/test_lib_storm.py +45 -0
  49. synapse/tests/test_lib_stormlib_aha.py +1 -2
  50. synapse/tests/test_lib_stormlib_auth.py +21 -0
  51. synapse/tests/test_lib_stormlib_cortex.py +12 -12
  52. synapse/tests/test_lib_stormlib_gen.py +99 -0
  53. synapse/tests/test_lib_stormlib_model.py +108 -0
  54. synapse/tests/test_lib_stormlib_modelext.py +64 -0
  55. synapse/tests/test_lib_stormlib_tabular.py +226 -0
  56. synapse/tests/test_lib_stormsvc.py +4 -1
  57. synapse/tests/test_lib_stormtypes.py +10 -0
  58. synapse/tests/test_model_base.py +3 -0
  59. synapse/tests/test_model_biz.py +3 -0
  60. synapse/tests/test_model_files.py +12 -2
  61. synapse/tests/test_model_inet.py +24 -0
  62. synapse/tests/test_tools_changelog.py +196 -0
  63. synapse/tests/test_tools_healthcheck.py +4 -3
  64. synapse/tests/utils.py +1 -1
  65. synapse/tools/changelog.py +774 -15
  66. {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/METADATA +3 -3
  67. {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/RECORD +70 -64
  68. {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/WHEEL +1 -1
  69. {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/LICENSE +0 -0
  70. {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py CHANGED
@@ -92,6 +92,7 @@ import synapse.lib.stormlib.scrape as s_stormlib_scrape # NOQA
92
92
  import synapse.lib.stormlib.infosec as s_stormlib_infosec # NOQA
93
93
  import synapse.lib.stormlib.project as s_stormlib_project # NOQA
94
94
  import synapse.lib.stormlib.spooled as s_stormlib_spooled # NOQA
95
+ import synapse.lib.stormlib.tabular as s_stormlib_tabular # NOQA
95
96
  import synapse.lib.stormlib.version as s_stormlib_version # NOQA
96
97
  import synapse.lib.stormlib.easyperm as s_stormlib_easyperm # NOQA
97
98
  import synapse.lib.stormlib.ethereum as s_stormlib_ethereum # NOQA
@@ -857,7 +858,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
857
858
  self.httpextapidb = self.slab.initdb('http:ext:apis')
858
859
 
859
860
  if self.inaugural:
860
- await self.cellinfo.set('cortex:version', s_version.version)
861
+ self.cellinfo.set('cortex:version', s_version.version)
861
862
 
862
863
  corevers = self.cellinfo.get('cortex:version')
863
864
  s_version.reqVersion(corevers, reqver, exc=s_exc.BadStorageVersion,
@@ -881,7 +882,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
881
882
 
882
883
  self.stormmods = {} # name: mdef
883
884
  self.stormpkgs = {} # name: pkgdef
884
- self.stormvars = None # type: s_hive.HiveDict
885
+ self.stormvars = None # type: s_lmdbslab.SafeKeyVal
885
886
 
886
887
  self.svcsbyiden = {}
887
888
  self.svcsbyname = {}
@@ -923,7 +924,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
923
924
  # generic fini handler for the Cortex
924
925
  self.onfini(self._onCoreFini)
925
926
 
926
- await self._initCoreHive()
927
+ self.cortexdata = self.slab.getSafeKeyVal('cortex')
928
+
929
+ await self._initCoreInfo()
927
930
  self._initStormLibs()
928
931
  self._initFeedFuncs()
929
932
 
@@ -943,6 +946,11 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
943
946
  (1, self._migrateTaxonomyIface),
944
947
  ), nexs=False)
945
948
 
949
+ await self._bumpCellVers('cortex:storage', (
950
+ (1, self._storUpdateMacros),
951
+ (4, self._storCortexHiveMigration),
952
+ ), nexs=False)
953
+
946
954
  # Perform module loading
947
955
  await self._loadCoreMods()
948
956
  await self._loadExtModel()
@@ -961,8 +969,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
961
969
 
962
970
  await self._initOAuthManager()
963
971
 
964
- stormdmonhive = await self.hive.open(('cortex', 'storm', 'dmons'))
965
- self.stormdmonhive = await stormdmonhive.dict()
972
+ self.stormdmondefs = self.cortexdata.getSubKeyVal('storm:dmons:')
966
973
  self.stormdmons = await s_storm.DmonManager.anit(self)
967
974
  self.onfini(self.stormdmons)
968
975
 
@@ -973,15 +980,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
973
980
 
974
981
  await self._initRuntFuncs()
975
982
 
976
- taghive = await self.hive.open(('cortex', 'tagmeta'))
977
- cmdhive = await self.hive.open(('cortex', 'storm', 'cmds'))
978
- pkghive = await self.hive.open(('cortex', 'storm', 'packages'))
979
- svchive = await self.hive.open(('cortex', 'storm', 'services'))
980
-
981
- self.taghive = await taghive.dict()
982
- self.cmdhive = await cmdhive.dict()
983
- self.pkghive = await pkghive.dict()
984
- self.svchive = await svchive.dict()
983
+ self.tagmeta = self.cortexdata.getSubKeyVal('tagmeta:')
984
+ self.cmddefs = self.cortexdata.getSubKeyVal('storm:cmds:')
985
+ self.pkgdefs = self.cortexdata.getSubKeyVal('storm:packages:')
986
+ self.svcdefs = self.cortexdata.getSubKeyVal('storm:services:')
985
987
 
986
988
  await self._initDeprLocks()
987
989
  await self._warnDeprLocks()
@@ -997,29 +999,87 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
997
999
 
998
1000
  # TODO - Remove this in 3.0.0
999
1001
  ag = await self.auth.addAuthGate('cortex', 'cortex')
1000
- for (useriden, user) in ag.gateusers.items():
1002
+ for useriden in ag.gateusers.keys():
1003
+ user = self.auth.user(useriden)
1004
+ if user is None:
1005
+ continue
1006
+
1001
1007
  mesg = f'User {useriden} ({user.name}) has a rule on the "cortex" authgate. This authgate is not used ' \
1002
1008
  f'for permission checks and will be removed in Synapse v3.0.0.'
1003
1009
  logger.warning(mesg, extra=await self.getLogExtra(user=useriden, username=user.name))
1004
- for (roleiden, role) in ag.gateroles.items():
1010
+ for roleiden in ag.gateroles.keys():
1011
+ role = self.auth.role(roleiden)
1012
+ if role is None:
1013
+ continue
1014
+
1005
1015
  mesg = f'Role {roleiden} ({role.name}) has a rule on the "cortex" authgate. This authgate is not used ' \
1006
1016
  f'for permission checks and will be removed in Synapse v3.0.0.'
1007
1017
  logger.warning(mesg, extra=await self.getLogExtra(role=roleiden, rolename=role.name))
1008
1018
 
1009
1019
  self._initVaults()
1010
1020
 
1011
- await self._bumpCellVers('cortex:storage', (
1012
- (1, self._storUpdateMacros),
1013
- (2, self._storLayrFeedDefaults),
1014
- (3, self._updateTriggerViewIdens),
1015
- ), nexs=False)
1021
+ async def _storCortexHiveMigration(self):
1022
+
1023
+ logger.warning('migrating Cortex data out of hive')
1024
+
1025
+ viewdefs = self.cortexdata.getSubKeyVal('view:info:')
1026
+ async with await self.hive.open(('cortex', 'views')) as viewnodes:
1027
+ for view_iden, node in viewnodes:
1028
+ viewdict = await node.dict()
1029
+ viewinfo = viewdict.pack()
1030
+ viewinfo.setdefault('iden', view_iden)
1031
+ viewdefs.set(view_iden, viewinfo)
1032
+
1033
+ trigdict = self.cortexdata.getSubKeyVal(f'view:{view_iden}:trigger:')
1034
+ async with await node.open(('triggers',)) as trignodes:
1035
+ for iden, trig in trignodes:
1036
+ valu = trig.valu
1037
+ if valu.get('view', s_common.novalu) != view_iden:
1038
+ valu['view'] = view_iden
1039
+ trigdict.set(iden, valu)
1040
+
1041
+ layrdefs = self.cortexdata.getSubKeyVal('layer:info:')
1042
+ async with await self.hive.open(('cortex', 'layers')) as layrnodes:
1043
+ for iden, node in layrnodes:
1044
+ layrdict = await node.dict()
1045
+ layrinfo = layrdict.pack()
1046
+ pushs = layrinfo.get('pushs', {})
1047
+ if pushs:
1048
+ for pdef in pushs.values():
1049
+ pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1050
+ pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1016
1051
 
1017
- async def _updateTriggerViewIdens(self):
1018
- for view in self.views.values():
1019
- for trigiden, trigger in await view.listTriggers():
1020
- if trigger.get('view') != view.iden:
1021
- trigger.tdef['view'] = view.iden
1022
- await view.trigdict.set(trigiden, trigger.tdef)
1052
+ pulls = layrinfo.get('pulls', {})
1053
+ if pulls:
1054
+ pulls = layrinfo.get('pulls', {})
1055
+ for pdef in pulls.values():
1056
+ pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1057
+ pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1058
+
1059
+ layrdefs.set(iden, layrinfo)
1060
+
1061
+ migrs = (
1062
+ (('agenda', 'appts'), 'agenda:appt:'),
1063
+ (('cortex', 'tagmeta'), 'tagmeta:'),
1064
+ (('cortex', 'storm', 'cmds'), 'storm:cmds:'),
1065
+ (('cortex', 'storm', 'vars'), 'storm:vars:'),
1066
+ (('cortex', 'storm', 'dmons'), 'storm:dmons:'),
1067
+ (('cortex', 'storm', 'packages'), 'storm:packages:'),
1068
+ (('cortex', 'storm', 'services'), 'storm:services:'),
1069
+ (('cortex', 'model', 'forms'), 'model:forms:'),
1070
+ (('cortex', 'model', 'props'), 'model:props:'),
1071
+ (('cortex', 'model', 'univs'), 'model:univs:'),
1072
+ (('cortex', 'model', 'tagprops'), 'model:tagprops:'),
1073
+ (('cortex', 'model', 'deprlocks'), 'model:deprlocks:'),
1074
+ )
1075
+
1076
+ for hivepath, kvpref in migrs:
1077
+ subkv = self.cortexdata.getSubKeyVal(kvpref)
1078
+ async with await self.hive.open(hivepath) as hivenode:
1079
+ for name, node in hivenode:
1080
+ subkv.set(name, node.valu)
1081
+
1082
+ logger.warning('...Cortex data migration complete!')
1023
1083
 
1024
1084
  async def _viewNomergeToProtected(self):
1025
1085
  for view in self.views.values():
@@ -1449,26 +1509,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1449
1509
  role = await self.auth.getRoleByName('all')
1450
1510
  await role.addRule((True, ('layer', 'read')), gateiden=layriden)
1451
1511
 
1452
- async def _storLayrFeedDefaults(self):
1453
-
1454
- for layer in list(self.layers.values()):
1455
- layrinfo = layer.layrinfo # type: s_hive.HiveDict
1456
-
1457
- pushs = layrinfo.get('pushs', {})
1458
- if pushs:
1459
- for pdef in pushs.values():
1460
- pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1461
- pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1462
- await layrinfo.set('pushs', pushs, nexs=False)
1463
-
1464
- pulls = layrinfo.get('pulls', {})
1465
- if pulls:
1466
- pulls = layrinfo.get('pulls', {})
1467
- for pdef in pulls.values():
1468
- pdef.setdefault('chunk:size', s_const.layer_pdef_csize)
1469
- pdef.setdefault('queue:size', s_const.layer_pdef_qsize)
1470
- await layrinfo.set('pulls', pulls, nexs=False)
1471
-
1472
1512
  async def initServiceRuntime(self):
1473
1513
 
1474
1514
  # do any post-nexus initialization here...
@@ -1598,8 +1638,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1598
1638
  mesg = 'setDeprLock() called on non-existant or non-deprecated form, property, or type.'
1599
1639
  raise s_exc.NoSuchProp(name=name, mesg=mesg)
1600
1640
 
1601
- self.deprlocks[name] = locked
1602
- await self.hive.set(('cortex', 'model', 'deprlocks'), self.deprlocks)
1641
+ self.deprlocks.set(name, locked)
1603
1642
 
1604
1643
  for elem in todo:
1605
1644
  elem.locked = locked
@@ -1905,14 +1944,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1905
1944
  await core.setTagModel("cno.cve", "regex", (None, None, "[0-9]{4}", "[0-9]{5}"))
1906
1945
 
1907
1946
  '''
1908
- meta = self.taghive.get(tagname)
1947
+ meta = self.tagmeta.get(tagname)
1909
1948
  if meta is None:
1910
1949
  meta = {}
1911
1950
 
1912
1951
  meta[name] = valu
1913
1952
  reqValidTagModel(meta)
1914
1953
 
1915
- await self.taghive.set(tagname, meta)
1954
+ self.tagmeta.set(tagname, meta)
1916
1955
 
1917
1956
  # clear cached entries
1918
1957
  if name == 'regex':
@@ -1928,7 +1967,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1928
1967
  Arguments:
1929
1968
  tagname (str): The name of the tag.
1930
1969
  '''
1931
- await self.taghive.pop(tagname)
1970
+ self.tagmeta.pop(tagname)
1932
1971
  self.tagvalid.clear()
1933
1972
  self.tagprune.clear()
1934
1973
 
@@ -1945,12 +1984,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1945
1984
  (object): The current value of the property.
1946
1985
  '''
1947
1986
 
1948
- meta = self.taghive.get(tagname)
1987
+ meta = self.tagmeta.get(tagname)
1949
1988
  if meta is None:
1950
1989
  return None
1951
1990
 
1952
1991
  retn = meta.pop(name, None)
1953
- await self.taghive.set(name, meta)
1992
+ self.tagmeta.set(tagname, meta)
1954
1993
 
1955
1994
  if name == 'regex':
1956
1995
  self.tagvalid.clear()
@@ -1973,7 +2012,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1973
2012
  parts = s_chop.tagpath(tagname)
1974
2013
  for tag in s_chop.tags(tagname):
1975
2014
 
1976
- meta = self.taghive.get(tag)
2015
+ meta = self.tagmeta.get(tag)
1977
2016
  if meta is None:
1978
2017
  continue
1979
2018
 
@@ -2006,7 +2045,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2006
2045
  prune.append(tag)
2007
2046
  continue
2008
2047
 
2009
- meta = self.taghive.get(tag)
2048
+ meta = self.tagmeta.get(tag)
2010
2049
  if meta is None:
2011
2050
  continue
2012
2051
 
@@ -2028,7 +2067,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2028
2067
  Returns:
2029
2068
  (dict): The tag model specification or None.
2030
2069
  '''
2031
- retn = self.taghive.get(tagname)
2070
+ retn = self.tagmeta.get(tagname)
2032
2071
  if retn is not None:
2033
2072
  return dict(retn)
2034
2073
 
@@ -2039,7 +2078,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2039
2078
  Returns:
2040
2079
  ([(str, dict), ...]): A list of tag model specification tuples.
2041
2080
  '''
2042
- return list(self.taghive.items())
2081
+ return list(self.tagmeta.items())
2043
2082
 
2044
2083
  async def _finiStor(self):
2045
2084
  await asyncio.gather(*[view.fini() for view in self.views.values()])
@@ -2085,7 +2124,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2085
2124
 
2086
2125
  async def _initStormDmons(self):
2087
2126
 
2088
- for iden, ddef in self.stormdmonhive.items():
2127
+ for iden, ddef in self.stormdmondefs.items():
2089
2128
  try:
2090
2129
  await self.runStormDmon(iden, ddef)
2091
2130
 
@@ -2097,7 +2136,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2097
2136
 
2098
2137
  async def _initStormSvcs(self):
2099
2138
 
2100
- for iden, sdef in self.svchive.items():
2139
+ for iden, sdef in self.svcdefs.items():
2101
2140
 
2102
2141
  try:
2103
2142
  await self._setStormSvc(sdef)
@@ -2160,7 +2199,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2160
2199
  '''
2161
2200
  name = cdef.get('name')
2162
2201
  await self._setStormCmd(cdef)
2163
- await self.cmdhive.set(name, cdef)
2202
+ self.cmddefs.set(name, cdef)
2164
2203
 
2165
2204
  async def _reqStormCmd(self, cdef):
2166
2205
 
@@ -2496,12 +2535,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2496
2535
  if ctor is None:
2497
2536
  return
2498
2537
 
2499
- cdef = self.cmdhive.get(name)
2538
+ cdef = self.cmddefs.get(name)
2500
2539
  if cdef is None:
2501
2540
  mesg = f'The storm command ({name}) is not dynamic.'
2502
2541
  raise s_exc.CantDelCmd(mesg=mesg)
2503
2542
 
2504
- await self.cmdhive.pop(name)
2543
+ self.cmddefs.pop(name)
2505
2544
  self.stormcmds.pop(name, None)
2506
2545
 
2507
2546
  await self.fire('core:cmd:change', cmd=name, act='del')
@@ -2556,7 +2595,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2556
2595
  @s_nexus.Pusher.onPush('pkg:add')
2557
2596
  async def _addStormPkg(self, pkgdef):
2558
2597
  name = pkgdef.get('name')
2559
- olddef = self.pkghive.get(name, None)
2598
+ olddef = self.pkgdefs.get(name, None)
2560
2599
  if olddef is not None:
2561
2600
  if s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
2562
2601
  await self._dropStormPkg(olddef)
@@ -2564,7 +2603,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2564
2603
  return
2565
2604
 
2566
2605
  await self.loadStormPkg(pkgdef)
2567
- await self.pkghive.set(name, pkgdef)
2606
+ self.pkgdefs.set(name, pkgdef)
2568
2607
 
2569
2608
  self._clearPermDefs()
2570
2609
 
@@ -2577,7 +2616,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2577
2616
  await self.feedBeholder('pkg:add', pkgdef, gates=gates, perms=perms)
2578
2617
 
2579
2618
  async def delStormPkg(self, name):
2580
- pkgdef = self.pkghive.get(name, None)
2619
+ pkgdef = self.pkgdefs.get(name)
2581
2620
  if pkgdef is None:
2582
2621
  mesg = f'No storm package: {name}.'
2583
2622
  raise s_exc.NoSuchPkg(mesg=mesg)
@@ -2589,7 +2628,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2589
2628
  '''
2590
2629
  Delete a storm package by name.
2591
2630
  '''
2592
- pkgdef = await self.pkghive.pop(name, None)
2631
+ pkgdef = self.pkgdefs.pop(name, None)
2593
2632
  if pkgdef is None:
2594
2633
  return
2595
2634
 
@@ -2612,7 +2651,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2612
2651
  return self._getStormPkgs()
2613
2652
 
2614
2653
  def _getStormPkgs(self):
2615
- return copy.deepcopy(list(self.pkghive.values()))
2654
+ return copy.deepcopy(list(self.pkgdefs.values()))
2616
2655
 
2617
2656
  async def getStormMods(self):
2618
2657
  return copy.deepcopy(self.stormmods)
@@ -2927,13 +2966,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2927
2966
  return ssvc.sdef
2928
2967
 
2929
2968
  ssvc = await self._setStormSvc(sdef)
2930
- await self.svchive.set(iden, sdef)
2969
+ self.svcdefs.set(iden, sdef)
2931
2970
 
2932
2971
  await self.feedBeholder('svc:add', {'name': sdef.get('name'), 'iden': iden})
2933
2972
  return ssvc.sdef
2934
2973
 
2935
2974
  async def delStormSvc(self, iden):
2936
- sdef = self.svchive.get(iden)
2975
+ sdef = self.svcdefs.get(iden)
2937
2976
  if sdef is None:
2938
2977
  mesg = f'No storm service with iden: {iden}'
2939
2978
  raise s_exc.NoSuchStormSvc(mesg=mesg, iden=iden)
@@ -2945,7 +2984,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2945
2984
  '''
2946
2985
  Delete a registered storm service from the cortex.
2947
2986
  '''
2948
- sdef = self.svchive.get(iden)
2987
+ sdef = self.svcdefs.get(iden)
2949
2988
  if sdef is None: # pragma: no cover
2950
2989
  return
2951
2990
 
@@ -2957,7 +2996,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2957
2996
  except Exception as e:
2958
2997
  logger.exception(f'service.del hook for service {iden} failed with error: {e}')
2959
2998
 
2960
- sdef = await self.svchive.pop(iden)
2999
+ sdef = self.svcdefs.pop(iden)
2961
3000
 
2962
3001
  await self._delStormSvcPkgs(iden)
2963
3002
 
@@ -2983,7 +3022,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
2983
3022
 
2984
3023
  def getStormSvcPkgs(self, iden):
2985
3024
  pkgs = []
2986
- for _, pdef in self.pkghive.items():
3025
+ for _, pdef in self.pkgdefs.items():
2987
3026
  pkgiden = pdef.get('svciden')
2988
3027
  if pkgiden and pkgiden == iden:
2989
3028
  pkgs.append(pdef)
@@ -3020,17 +3059,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3020
3059
  Returns:
3021
3060
  dict: An updated storm service definition dictionary.
3022
3061
  '''
3023
- sdef = self.svchive.get(iden)
3062
+ sdef = self.svcdefs.get(iden)
3024
3063
  if sdef is None:
3025
3064
  mesg = f'No storm service with iden: {iden}'
3026
3065
  raise s_exc.NoSuchStormSvc(mesg=mesg)
3027
3066
 
3028
3067
  sdef['evts'] = edef
3029
- await self.svchive.set(iden, sdef)
3068
+ self.svcdefs.set(iden, sdef)
3030
3069
  return sdef
3031
3070
 
3032
3071
  async def _runStormSvcAdd(self, iden):
3033
- sdef = self.svchive.get(iden)
3072
+ sdef = self.svcdefs.get(iden)
3034
3073
  if sdef is None:
3035
3074
  mesg = f'No storm service with iden: {iden}'
3036
3075
  raise s_exc.NoSuchStormSvc(mesg=mesg)
@@ -3047,12 +3086,12 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3047
3086
  return
3048
3087
 
3049
3088
  sdef['added'] = True
3050
- await self.svchive.set(iden, sdef)
3089
+ self.svcdefs.set(iden, sdef)
3051
3090
 
3052
3091
  async def runStormSvcEvent(self, iden, name):
3053
3092
  assert name in ('add', 'del')
3054
3093
 
3055
- sdef = self.svchive.get(iden)
3094
+ sdef = self.svcdefs.get(iden)
3056
3095
  if sdef is None:
3057
3096
  mesg = f'No storm service with iden: {iden}'
3058
3097
  raise s_exc.NoSuchStormSvc(mesg=mesg)
@@ -3085,15 +3124,15 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3085
3124
  # Global stormvars APIs
3086
3125
 
3087
3126
  async def getStormVar(self, name, default=None):
3088
- return self.stormvars.get(name, default=default)
3127
+ return self.stormvars.get(name, defv=default)
3089
3128
 
3090
3129
  @s_nexus.Pusher.onPushAuto('stormvar:pop')
3091
3130
  async def popStormVar(self, name, default=None):
3092
- return await self.stormvars.pop(name, default=default)
3131
+ return self.stormvars.pop(name, defv=default)
3093
3132
 
3094
3133
  @s_nexus.Pusher.onPushAuto('stormvar:set')
3095
3134
  async def setStormVar(self, name, valu):
3096
- return await self.stormvars.set(name, valu)
3135
+ return self.stormvars.set(name, valu)
3097
3136
 
3098
3137
  async def itemsStormVar(self):
3099
3138
  for item in self.stormvars.items():
@@ -3125,17 +3164,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3125
3164
 
3126
3165
  async def _loadExtModel(self):
3127
3166
 
3128
- self.extforms = await (await self.hive.open(('cortex', 'model', 'forms'))).dict()
3129
- self.extprops = await (await self.hive.open(('cortex', 'model', 'props'))).dict()
3130
- self.extunivs = await (await self.hive.open(('cortex', 'model', 'univs'))).dict()
3131
- self.exttagprops = await (await self.hive.open(('cortex', 'model', 'tagprops'))).dict()
3167
+ self.extforms = self.cortexdata.getSubKeyVal('model:forms:')
3168
+ self.extprops = self.cortexdata.getSubKeyVal('model:props:')
3169
+ self.extunivs = self.cortexdata.getSubKeyVal('model:univs:')
3170
+ self.extedges = self.cortexdata.getSubKeyVal('model:edges:')
3171
+ self.exttagprops = self.cortexdata.getSubKeyVal('model:tagprops:')
3132
3172
 
3133
3173
  for formname, basetype, typeopts, typeinfo in self.extforms.values():
3134
3174
  try:
3135
3175
  self.model.addType(formname, basetype, typeopts, typeinfo)
3136
3176
  form = self.model.addForm(formname, {}, ())
3137
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3138
- raise
3139
3177
  except Exception as e:
3140
3178
  logger.warning(f'Extended form ({formname}) error: {e}')
3141
3179
  else:
@@ -3147,8 +3185,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3147
3185
  for form, prop, tdef, info in self.extprops.values():
3148
3186
  try:
3149
3187
  prop = self.model.addFormProp(form, prop, tdef, info)
3150
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3151
- raise
3152
3188
  except Exception as e:
3153
3189
  logger.warning(f'ext prop ({form}:{prop}) error: {e}')
3154
3190
  else:
@@ -3160,19 +3196,21 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3160
3196
  for prop, tdef, info in self.extunivs.values():
3161
3197
  try:
3162
3198
  self.model.addUnivProp(prop, tdef, info)
3163
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3164
- raise
3165
3199
  except Exception as e:
3166
3200
  logger.warning(f'ext univ ({prop}) error: {e}')
3167
3201
 
3168
3202
  for prop, tdef, info in self.exttagprops.values():
3169
3203
  try:
3170
3204
  self.model.addTagProp(prop, tdef, info)
3171
- except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
3172
- raise
3173
3205
  except Exception as e:
3174
3206
  logger.warning(f'ext tag prop ({prop}) error: {e}')
3175
3207
 
3208
+ for edge, info in self.extedges.values():
3209
+ try:
3210
+ self.model.addEdge(edge, info)
3211
+ except Exception as e:
3212
+ logger.warning(f'ext edge ({edge}) error: {e}')
3213
+
3176
3214
  async def getExtModel(self):
3177
3215
  '''
3178
3216
  Get all extended model properties in the Cortex.
@@ -3192,6 +3230,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3192
3230
 
3193
3231
  for prop, tdef, info in self.exttagprops.values():
3194
3232
  ret['tagprops'].append((prop, tdef, info))
3233
+
3234
+ for edge, info in self.extedges.values():
3235
+ ret['edges'].append((edge, info))
3236
+
3195
3237
  ret['version'] = (1, 0)
3196
3238
  return copy.deepcopy(dict(ret))
3197
3239
 
@@ -3206,9 +3248,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3206
3248
  Bool: True when the model was added.
3207
3249
 
3208
3250
  Raises:
3209
- s_exc.BadFormDef: If a form exists with a different definition the provided definition.
3210
- s_exc.BadPropDef: If a propery, tagprop, or universal propert from exists with a different definition
3251
+ s_exc.BadFormDef: If a form exists with a different definition than the provided definition.
3252
+ s_exc.BadPropDef: If a property, tagprop, or universal property exists with a different definition
3211
3253
  than the provided definition.
3254
+ s_exc.BadEdgeDef: If an edge exists with a different definition than the provided definition.
3212
3255
  '''
3213
3256
 
3214
3257
  # Get our current model definition
@@ -3219,11 +3262,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3219
3262
  props = {(info[0], info[1]): info for info in model.get('props', ())}
3220
3263
  tagprops = {info[0]: info for info in model.get('tagprops', ())}
3221
3264
  univs = {info[0]: info for info in model.get('univs', ())}
3265
+ edges = {info[0]: info for info in model.get('edges', ())}
3222
3266
 
3223
3267
  efrms = {info[0]: info for info in emodl.get('forms', ())}
3224
3268
  eprops = {(info[0], info[1]): info for info in emodl.get('props', ())}
3225
3269
  etagprops = {info[0]: info for info in emodl.get('tagprops', ())}
3226
3270
  eunivs = {info[0]: info for info in emodl.get('univs', ())}
3271
+ eedges = {info[0]: info for info in emodl.get('edges', ())}
3227
3272
 
3228
3273
  for (name, info) in forms.items():
3229
3274
  enfo = efrms.get(name)
@@ -3233,7 +3278,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3233
3278
  if enfo == info:
3234
3279
  continue
3235
3280
  mesg = f'Extended form definition differs from existing definition for {name}.'
3236
- raise s_exc.BadFormDef(mesg)
3281
+ raise s_exc.BadFormDef(mesg=mesg, name=name)
3237
3282
 
3238
3283
  for (name, info) in props.items():
3239
3284
  enfo = eprops.get(name)
@@ -3243,7 +3288,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3243
3288
  if enfo == info:
3244
3289
  continue
3245
3290
  mesg = f'Extended prop definition differs from existing definition for {name}'
3246
- raise s_exc.BadPropDef(mesg)
3291
+ raise s_exc.BadPropDef(mesg=mesg, name=name)
3247
3292
 
3248
3293
  for (name, info) in tagprops.items():
3249
3294
  enfo = etagprops.get(name)
@@ -3253,7 +3298,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3253
3298
  if enfo == info:
3254
3299
  continue
3255
3300
  mesg = f'Extended tagprop definition differs from existing definition for {name}'
3256
- raise s_exc.BadPropDef(mesg)
3301
+ raise s_exc.BadPropDef(mesg=mesg, name=name)
3257
3302
 
3258
3303
  for (name, info) in univs.items():
3259
3304
  enfo = eunivs.get(name)
@@ -3262,8 +3307,20 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3262
3307
  continue
3263
3308
  if enfo == info:
3264
3309
  continue
3265
- mesg = f'Extended universal poroperty definition differs from existing definition for {name}'
3266
- raise s_exc.BadPropDef(mesg)
3310
+ mesg = f'Extended universal property definition differs from existing definition for {name}'
3311
+ raise s_exc.BadPropDef(mesg=mesg, name=name)
3312
+
3313
+ for (name, info) in edges.items():
3314
+ enfo = eedges.get(name)
3315
+ if enfo is None:
3316
+ amodl['edges'].append(info)
3317
+ continue
3318
+ if enfo == info:
3319
+ continue
3320
+
3321
+ (n1form, verb, n2form) = info[0]
3322
+ mesg = f'Extended edge definition differs from existing definition for {info[0]}'
3323
+ raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
3267
3324
 
3268
3325
  for formname, basetype, typeopts, typeinfo in amodl['forms']:
3269
3326
  await self.addForm(formname, basetype, typeopts, typeinfo)
@@ -3277,6 +3334,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3277
3334
  for prop, tdef, info in amodl['univs']:
3278
3335
  await self.addUnivProp(prop, tdef, info)
3279
3336
 
3337
+ for edge, info in amodl['edges']:
3338
+ await self.addEdge(edge, info)
3339
+
3280
3340
  return True
3281
3341
 
3282
3342
  async def addUnivProp(self, name, tdef, info):
@@ -3307,7 +3367,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3307
3367
 
3308
3368
  self.model.addUnivProp(name, tdef, info)
3309
3369
 
3310
- await self.extunivs.set(name, (name, tdef, info))
3370
+ self.extunivs.set(name, (name, tdef, info))
3311
3371
  await self.fire('core:extmodel:change', prop=name, act='add', type='univ')
3312
3372
  base = '.' + name
3313
3373
  univ = self.model.univ(base)
@@ -3349,7 +3409,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3349
3409
  self.model.addType(formname, basetype, typeopts, typeinfo)
3350
3410
  self.model.addForm(formname, {}, ())
3351
3411
 
3352
- await self.extforms.set(formname, (formname, basetype, typeopts, typeinfo))
3412
+ self.extforms.set(formname, (formname, basetype, typeopts, typeinfo))
3353
3413
  await self.fire('core:extmodel:change', form=formname, act='add', type='form')
3354
3414
  form = self.model.form(formname)
3355
3415
  ftyp = self.model.type(formname)
@@ -3379,7 +3439,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3379
3439
  self.model.delForm(formname)
3380
3440
  self.model.delType(formname)
3381
3441
 
3382
- await self.extforms.pop(formname, None)
3442
+ self.extforms.pop(formname, None)
3383
3443
  await self.fire('core:extmodel:change', form=formname, act='del', type='form')
3384
3444
  await self.feedBeholder('model:form:del', {'form': formname})
3385
3445
 
@@ -3415,7 +3475,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3415
3475
  logger.warning(mesg)
3416
3476
 
3417
3477
  full = f'{form}:{prop}'
3418
- await self.extprops.set(full, (form, prop, tdef, info))
3478
+ self.extprops.set(full, (form, prop, tdef, info))
3419
3479
  await self.fire('core:extmodel:change', form=form, prop=prop, act='add', type='formprop')
3420
3480
  prop = self.model.prop(full)
3421
3481
  if prop:
@@ -3448,7 +3508,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3448
3508
  raise s_exc.CantDelProp(mesg=mesg)
3449
3509
 
3450
3510
  self.model.delFormProp(form, prop)
3451
- await self.extprops.pop(full, None)
3511
+ self.extprops.pop(full, None)
3452
3512
  await self.fire('core:extmodel:change',
3453
3513
  form=form, prop=prop, act='del', type='formprop')
3454
3514
 
@@ -3478,7 +3538,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3478
3538
  raise s_exc.CantDelUniv(mesg=mesg)
3479
3539
 
3480
3540
  self.model.delUnivProp(prop)
3481
- await self.extunivs.pop(prop, None)
3541
+ self.extunivs.pop(prop, None)
3482
3542
  await self.fire('core:extmodel:change', name=prop, act='del', type='univ')
3483
3543
  await self.feedBeholder('model:univ:del', {'prop': univname})
3484
3544
 
@@ -3503,7 +3563,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3503
3563
 
3504
3564
  self.model.addTagProp(name, tdef, info)
3505
3565
 
3506
- await self.exttagprops.set(name, (name, tdef, info))
3566
+ self.exttagprops.set(name, (name, tdef, info))
3507
3567
  await self.fire('core:tagprop:change', name=name, act='add')
3508
3568
  tagp = self.model.tagprop(name)
3509
3569
  if tagp:
@@ -3530,10 +3590,60 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3530
3590
 
3531
3591
  self.model.delTagProp(name)
3532
3592
 
3533
- await self.exttagprops.pop(name, None)
3593
+ self.exttagprops.pop(name, None)
3534
3594
  await self.fire('core:tagprop:change', name=name, act='del')
3535
3595
  await self.feedBeholder('model:tagprop:del', {'tagprop': name})
3536
3596
 
3597
+ async def addEdge(self, edge, edgeinfo):
3598
+ if not isinstance(edgeinfo, dict):
3599
+ mesg = 'Edge info should be a dict.'
3600
+ raise s_exc.BadArg(mesg=mesg, edgeinfo=edgeinfo)
3601
+
3602
+ (n1form, verb, n2form) = edge
3603
+ if not verb.startswith('_'):
3604
+ mesg = f'Extended edge verb must begin with "_"; got {verb}'
3605
+ raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
3606
+
3607
+ if n1form is not None:
3608
+ self.model._reqFormName(n1form)
3609
+
3610
+ if n2form is not None:
3611
+ self.model._reqFormName(n2form)
3612
+
3613
+ if self.model.edge(edge) is not None:
3614
+ raise s_exc.DupEdgeType.init(edge)
3615
+
3616
+ return await self._push('model:edge:add', edge, edgeinfo)
3617
+
3618
+ @s_nexus.Pusher.onPush('model:edge:add')
3619
+ async def _addEdge(self, edge, edgeinfo):
3620
+ if self.model.edge(edge) is not None:
3621
+ return
3622
+
3623
+ self.model.addEdge(edge, edgeinfo)
3624
+
3625
+ self.extedges.set(s_common.guid(edge), (edge, edgeinfo))
3626
+ await self.fire('core:extmodel:change', edge=edge, act='add', type='edge')
3627
+ await self.feedBeholder('model:edge:add', {'edge': edge, 'info': edgeinfo})
3628
+
3629
+ async def delEdge(self, edge):
3630
+ if self.extedges.get(s_common.guid(edge)) is None:
3631
+ raise s_exc.NoSuchEdge.init(edge)
3632
+
3633
+ return await self._push('model:edge:del', edge)
3634
+
3635
+ @s_nexus.Pusher.onPush('model:edge:del')
3636
+ async def _delEdge(self, edge):
3637
+ edgeguid = s_common.guid(edge)
3638
+ if self.extedges.get(edgeguid) is None:
3639
+ return
3640
+
3641
+ self.model.delEdge(edge)
3642
+
3643
+ self.extedges.pop(edgeguid, None)
3644
+ await self.fire('core:extmodel:change', edge=edge, act='del', type='edge')
3645
+ await self.feedBeholder('model:edge:del', {'edge': edge})
3646
+
3537
3647
  async def addNodeTag(self, user, iden, tag, valu=(None, None)):
3538
3648
  '''
3539
3649
  Add a tag to a node specified by iden.
@@ -3821,15 +3931,13 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
3821
3931
 
3822
3932
  await self.waitfini(1)
3823
3933
 
3824
- async def _initCoreHive(self):
3825
- stormvarsnode = await self.hive.open(('cortex', 'storm', 'vars'))
3826
- self.stormvars = await stormvarsnode.dict()
3934
+ async def _initCoreInfo(self):
3935
+ self.stormvars = self.cortexdata.getSubKeyVal('storm:vars:')
3827
3936
  if self.inaugural:
3828
- await self.stormvars.set(s_stormlib_cell.runtime_fixes_key, s_stormlib_cell.getMaxHotFixes())
3829
- self.onfini(self.stormvars)
3937
+ self.stormvars.set(s_stormlib_cell.runtime_fixes_key, s_stormlib_cell.getMaxHotFixes())
3830
3938
 
3831
3939
  async def _initDeprLocks(self):
3832
- self.deprlocks = await self.hive.get(('cortex', 'model', 'deprlocks'), {}) # type: s_hive.Node
3940
+ self.deprlocks = self.cortexdata.getSubKeyVal('model:deprlocks:')
3833
3941
  # TODO: 3.0.0 conversion will truncate this hive key
3834
3942
 
3835
3943
  if self.inaugural:
@@ -4062,18 +4170,18 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4062
4170
 
4063
4171
  async def _initPureStormCmds(self):
4064
4172
  oldcmds = []
4065
- for name, cdef in self.cmdhive.items():
4173
+ for name, cdef in self.cmddefs.items():
4066
4174
  cmdiden = cdef.get('cmdconf', {}).get('svciden')
4067
- if cmdiden and self.svchive.get(cmdiden) is None:
4175
+ if cmdiden and self.svcdefs.get(cmdiden) is None:
4068
4176
  oldcmds.append(name)
4069
4177
  else:
4070
4178
  await self._trySetStormCmd(name, cdef)
4071
4179
 
4072
4180
  for name in oldcmds:
4073
4181
  logger.warning(f'Removing old command: [{name}]')
4074
- await self.cmdhive.pop(name)
4182
+ self.cmddefs.pop(name)
4075
4183
 
4076
- for pkgdef in self.pkghive.values():
4184
+ for pkgdef in self.pkgdefs.values():
4077
4185
  await self._tryLoadStormPkg(pkgdef)
4078
4186
 
4079
4187
  async def _trySetStormCmd(self, name, cdef):
@@ -4399,9 +4507,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4399
4507
  mrev = s_modelrev.ModelRev(self)
4400
4508
  await mrev.revCoreLayers()
4401
4509
 
4402
- async def _loadView(self, node):
4510
+ async def _loadView(self, vdef):
4403
4511
 
4404
- view = await self.viewctor(self, node)
4512
+ view = await self.viewctor(self, vdef)
4405
4513
 
4406
4514
  self.views[view.iden] = view
4407
4515
  self.dynitems[view.iden] = view
@@ -4425,8 +4533,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4425
4533
 
4426
4534
  defiden = self.cellinfo.get('defaultview')
4427
4535
 
4428
- for iden, node in await self.hive.open(('cortex', 'views')):
4429
- view = await self._loadView(node)
4536
+ self.viewdefs = self.cortexdata.getSubKeyVal('view:info:')
4537
+
4538
+ for iden, vdef in self.viewdefs.items():
4539
+ view = await self._loadView(vdef)
4430
4540
  if iden == defiden:
4431
4541
  self.view = view
4432
4542
 
@@ -4450,7 +4560,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4450
4560
  }
4451
4561
  vdef = await self.addView(vdef, nexs=False)
4452
4562
  iden = vdef.get('iden')
4453
- await self.cellinfo.set('defaultview', iden)
4563
+ self.cellinfo.set('defaultview', iden)
4454
4564
  self.view = self.getView(iden)
4455
4565
 
4456
4566
  self._calcViewsByLayer()
@@ -4497,13 +4607,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4497
4607
  role = await self.auth.getRoleByName('all')
4498
4608
  await role.addRule((True, ('view', 'read')), gateiden=iden, nexs=False)
4499
4609
 
4500
- node = await self.hive.open(('cortex', 'views', iden))
4501
-
4502
- info = await node.dict()
4503
- for name, valu in vdef.items():
4504
- await info.set(name, valu)
4610
+ self.viewdefs.set(iden, vdef)
4505
4611
 
4506
- view = await self._loadView(node)
4612
+ view = await self._loadView(vdef)
4507
4613
  view.init2()
4508
4614
 
4509
4615
  self._calcViewsByLayer()
@@ -4546,7 +4652,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4546
4652
 
4547
4653
  if (view := self.views.get(viewiden)) is not None:
4548
4654
 
4549
- await self.hive.pop(('cortex', 'views', viewiden))
4655
+ self.viewdefs.pop(viewiden)
4550
4656
  await view.delete()
4551
4657
 
4552
4658
  self._calcViewsByLayer()
@@ -4563,8 +4669,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4563
4669
  continue
4564
4670
 
4565
4671
  view.layers = [lyr for lyr in view.layers if lyr.iden != layriden]
4672
+
4566
4673
  layridens = [lyr.iden for lyr in view.layers]
4567
- await view.info.set('layers', layridens)
4674
+ view.info['layers'] = layridens
4568
4675
 
4569
4676
  mesg = {'iden': view.iden, 'layers': layridens}
4570
4677
  await self.feedBeholder('view:setlayers', mesg, gates=[view.iden, layridens[0]])
@@ -4572,14 +4679,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4572
4679
  if view.parent.iden == viewiden:
4573
4680
  if newparent is None:
4574
4681
  view.parent = None
4575
- await view.info.pop('parent')
4682
+ view.info['parent'] = None
4576
4683
  else:
4577
4684
  view.parent = newview
4578
- await view.info.set('parent', newparent)
4685
+ view.info['parent'] = newparent
4579
4686
 
4580
4687
  mesg = {'iden': view.iden, 'name': 'parent', 'valu': newparent}
4581
4688
  await self.feedBeholder('view:set', mesg, gates=[view.iden, layridens[0]])
4582
4689
 
4690
+ self.viewdefs.set(view.iden, view.info)
4691
+
4583
4692
  if not layrinuse and (layr := self.layers.get(layriden)) is not None:
4584
4693
  del self.layers[layriden]
4585
4694
 
@@ -4593,8 +4702,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4593
4702
  await self.auth.delAuthGate(layriden)
4594
4703
  self.dynitems.pop(layriden)
4595
4704
 
4596
- await self.hive.pop(('cortex', 'layers', layriden))
4597
-
4705
+ self.layerdefs.pop(layriden)
4598
4706
  await layr.delete()
4599
4707
 
4600
4708
  layr.deloffs = nexsitem[0]
@@ -4629,7 +4737,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4629
4737
  if cview.parent is not None and cview.parent.iden == iden:
4630
4738
  raise s_exc.SynErr(mesg='Cannot delete a view that has children')
4631
4739
 
4632
- await self.hive.pop(('cortex', 'views', iden))
4740
+ self.viewdefs.pop(iden)
4633
4741
  await view.delete()
4634
4742
 
4635
4743
  self._calcViewsByLayer()
@@ -4665,7 +4773,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4665
4773
  await self.auth.delAuthGate(iden)
4666
4774
  self.dynitems.pop(iden)
4667
4775
 
4668
- await self.hive.pop(('cortex', 'layers', iden))
4776
+ self.layerdefs.pop(iden)
4669
4777
 
4670
4778
  await layr.delete()
4671
4779
 
@@ -4811,6 +4919,60 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4811
4919
  else:
4812
4920
  return await self._addLayer(ldef, (None, None))
4813
4921
 
4922
+ async def _twinLayer(self, oldlayr):
4923
+
4924
+ newldef = s_msgpack.deepcopy(oldlayr.layrinfo)
4925
+
4926
+ newldef.pop('iden', None)
4927
+
4928
+ newldef = await self.addLayer(newldef)
4929
+ newlayr = self.reqLayer(newldef.get('iden'))
4930
+
4931
+ oldinfo = self.auth.reqAuthGate(oldlayr.iden).pack()
4932
+
4933
+ for userinfo in oldinfo.get('users', ()):
4934
+
4935
+ user = self.auth.user(userinfo.get('iden'))
4936
+ if user is None: # pragma: no cover
4937
+ continue
4938
+
4939
+ if userinfo.get('admin'):
4940
+ await user.setAdmin(True, gateiden=newlayr.iden)
4941
+
4942
+ for rule in userinfo.get('rules', ()):
4943
+ await user.addRule(rule, gateiden=newlayr.iden)
4944
+
4945
+ for roleinfo in oldinfo.get('roles', ()):
4946
+
4947
+ role = self.auth.role(roleinfo.get('iden'))
4948
+ if role is None: # pragma: no cover
4949
+ continue
4950
+
4951
+ for rule in roleinfo.get('rules', ()):
4952
+ await role.addRule(rule, gateiden=newlayr.iden)
4953
+
4954
+ return newlayr
4955
+
4956
+ @s_nexus.Pusher.onPushAuto('layer:swap')
4957
+ async def swapLayer(self, oldiden, newiden):
4958
+ '''
4959
+ Atomically swap out a layer from all views that contain it.
4960
+ '''
4961
+ self.reqLayer(oldiden)
4962
+ self.reqLayer(newiden)
4963
+
4964
+ for view in list(self.views.values()):
4965
+ await asyncio.sleep(0)
4966
+
4967
+ oldlayers = view.info.get('layers')
4968
+ if oldiden not in oldlayers:
4969
+ continue
4970
+
4971
+ newlayers = list(oldlayers)
4972
+ newlayers[oldlayers.index(oldiden)] = newiden
4973
+
4974
+ await view._setLayerIdens(newlayers)
4975
+
4814
4976
  @s_nexus.Pusher.onPush('layer:add', passitem=True)
4815
4977
  async def _addLayer(self, ldef, nexsitem):
4816
4978
 
@@ -4827,13 +4989,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4827
4989
 
4828
4990
  user = await self.auth.reqUser(creator)
4829
4991
 
4830
- node = await self.hive.open(('cortex', 'layers', iden))
4992
+ self.layerdefs.set(iden, ldef)
4831
4993
 
4832
- layrinfo = await node.dict()
4833
- for name, valu in ldef.items():
4834
- await layrinfo.set(name, valu)
4835
-
4836
- layr = await self._initLayr(layrinfo, nexsoffs=nexsitem[0])
4994
+ layr = await self._initLayr(ldef, nexsoffs=nexsitem[0])
4837
4995
  await user.setAdmin(True, gateiden=iden, logged=False)
4838
4996
 
4839
4997
  # forward wind the new layer to the current model version
@@ -4899,10 +5057,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4899
5057
  return await s_layer.Layer.anit(self, layrinfo)
4900
5058
 
4901
5059
  async def _initCoreLayers(self):
4902
- node = await self.hive.open(('cortex', 'layers'))
4903
- for _, node in node:
4904
- layrinfo = await node.dict()
4905
- await self._initLayr(layrinfo)
5060
+ self.layerdefs = self.cortexdata.getSubKeyVal('layer:info:')
5061
+ for ldef in self.layerdefs.values():
5062
+ await self._initLayr(ldef)
4906
5063
 
4907
5064
  @s_nexus.Pusher.onPushAuto('layer:push:add')
4908
5065
  async def addLayrPush(self, layriden, pdef):
@@ -4925,7 +5082,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4925
5082
 
4926
5083
  pushs[iden] = pdef
4927
5084
 
4928
- await layr.layrinfo.set('pushs', pushs)
5085
+ layr.layrinfo['pushs'] = pushs
5086
+ self.layerdefs.set(layr.iden, layr.layrinfo)
5087
+
4929
5088
  await self.runLayrPush(layr, pdef)
4930
5089
 
4931
5090
  @s_nexus.Pusher.onPushAuto('layer:push:del')
@@ -4943,7 +5102,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4943
5102
  if pdef is None:
4944
5103
  return
4945
5104
 
4946
- await layr.layrinfo.set('pushs', pushs)
5105
+ layr.layrinfo['pushs'] = pushs
5106
+ self.layerdefs.set(layr.iden, layr.layrinfo)
5107
+
4947
5108
  await self.delActiveCoro(pushiden)
4948
5109
 
4949
5110
  @s_nexus.Pusher.onPushAuto('layer:pull:add')
@@ -4966,7 +5127,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4966
5127
  return
4967
5128
 
4968
5129
  pulls[iden] = pdef
4969
- await layr.layrinfo.set('pulls', pulls)
5130
+
5131
+ layr.layrinfo['pulls'] = pulls
5132
+ self.layerdefs.set(layr.iden, layr.layrinfo)
4970
5133
 
4971
5134
  await self.runLayrPull(layr, pdef)
4972
5135
 
@@ -4985,7 +5148,9 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
4985
5148
  if pdef is None:
4986
5149
  return
4987
5150
 
4988
- await layr.layrinfo.set('pulls', pulls)
5151
+ layr.layrinfo['pulls'] = pulls
5152
+ self.layerdefs.set(layr.iden, layr.layrinfo)
5153
+
4989
5154
  await self.delActiveCoro(pulliden)
4990
5155
 
4991
5156
  async def runLayrPush(self, layr, pdef):
@@ -5106,24 +5271,20 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5106
5271
  newpath = s_common.gendir(self.dirn, 'layers', newiden)
5107
5272
  await layr.clone(newpath)
5108
5273
 
5109
- node = await self.hive.open(('cortex', 'layers', iden))
5110
- copynode = await self.hive.open(('cortex', 'layers', newiden))
5274
+ copyinfo = self.layerdefs.get(iden)
5111
5275
 
5112
- layrinfo = await node.dict()
5113
- copyinfo = await copynode.dict()
5114
- for name, valu in layrinfo.items():
5115
- await copyinfo.set(name, valu)
5276
+ for name, valu in copyinfo.items():
5277
+ ldef.setdefault(name, valu)
5116
5278
 
5117
- for name, valu in ldef.items():
5118
- await copyinfo.set(name, valu)
5279
+ self.layerdefs.set(newiden, ldef)
5119
5280
 
5120
- copylayr = await self._initLayr(copyinfo, nexsoffs=nexsitem[0])
5281
+ copylayr = await self._initLayr(ldef, nexsoffs=nexsitem[0])
5121
5282
 
5122
5283
  creator = copyinfo.get('creator')
5123
5284
  user = await self.auth.reqUser(creator)
5124
5285
  await user.setAdmin(True, gateiden=newiden, logged=False)
5125
5286
 
5126
- return await copylayr.pack()
5287
+ return ldef
5127
5288
 
5128
5289
  def addStormCmd(self, ctor):
5129
5290
  '''
@@ -5149,7 +5310,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5149
5310
 
5150
5311
  @s_nexus.Pusher.onPushAuto('storm:dmon:bump')
5151
5312
  async def bumpStormDmon(self, iden):
5152
- ddef = self.stormdmonhive.get(iden)
5313
+ ddef = self.stormdmondefs.get(iden)
5153
5314
  if ddef is None:
5154
5315
  return False
5155
5316
 
@@ -5166,7 +5327,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5166
5327
  Args:
5167
5328
  iden (str): User iden.
5168
5329
  '''
5169
- for dmoniden, ddef in list(self.stormdmonhive.items()):
5330
+ for dmoniden, ddef in list(self.stormdmondefs.items()):
5170
5331
  if ddef.get('user') == iden:
5171
5332
  await self.bumpStormDmon(dmoniden)
5172
5333
 
@@ -5182,7 +5343,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5182
5343
  dmon.enabled = True
5183
5344
  dmon.ddef['enabled'] = True
5184
5345
 
5185
- await self.stormdmonhive.set(iden, dmon.ddef)
5346
+ self.stormdmondefs.set(iden, dmon.ddef)
5186
5347
 
5187
5348
  if self.isactive:
5188
5349
  await dmon.run()
@@ -5202,7 +5363,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5202
5363
  dmon.enabled = False
5203
5364
  dmon.ddef['enabled'] = False
5204
5365
 
5205
- await self.stormdmonhive.set(iden, dmon.ddef)
5366
+ self.stormdmondefs.set(iden, dmon.ddef)
5206
5367
 
5207
5368
  if self.isactive:
5208
5369
  await dmon.stop()
@@ -5223,14 +5384,14 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5223
5384
 
5224
5385
  dmon = await self.runStormDmon(iden, ddef)
5225
5386
 
5226
- await self.stormdmonhive.set(iden, ddef)
5387
+ self.stormdmondefs.set(iden, ddef)
5227
5388
  return dmon.pack()
5228
5389
 
5229
5390
  async def delStormDmon(self, iden):
5230
5391
  '''
5231
5392
  Stop and remove a storm dmon.
5232
5393
  '''
5233
- ddef = self.stormdmonhive.get(iden)
5394
+ ddef = self.stormdmondefs.get(iden)
5234
5395
  if ddef is None:
5235
5396
  mesg = f'No storm daemon exists with iden {iden}.'
5236
5397
  raise s_exc.NoSuchIden(mesg=mesg)
@@ -5239,7 +5400,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5239
5400
 
5240
5401
  @s_nexus.Pusher.onPush('storm:dmon:del')
5241
5402
  async def _delStormDmon(self, iden):
5242
- ddef = await self.stormdmonhive.pop(iden)
5403
+ ddef = self.stormdmondefs.pop(iden)
5243
5404
  if ddef is None: # pragma: no cover
5244
5405
  return
5245
5406
  await self.stormdmons.popDmon(iden)