synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (89) hide show
  1. synapse/axon.py +3 -3
  2. synapse/cmds/cortex.py +1 -6
  3. synapse/common.py +7 -1
  4. synapse/cortex.py +145 -192
  5. synapse/datamodel.py +36 -1
  6. synapse/lib/agenda.py +87 -97
  7. synapse/lib/aha.py +51 -0
  8. synapse/lib/ast.py +22 -23
  9. synapse/lib/base.py +0 -6
  10. synapse/lib/boss.py +3 -0
  11. synapse/lib/cell.py +70 -39
  12. synapse/lib/certdir.py +9 -0
  13. synapse/lib/hiveauth.py +65 -12
  14. synapse/lib/httpapi.py +1 -0
  15. synapse/lib/modelrev.py +121 -33
  16. synapse/lib/modules.py +1 -0
  17. synapse/lib/nexus.py +64 -26
  18. synapse/lib/parser.py +2 -0
  19. synapse/lib/schemas.py +14 -0
  20. synapse/lib/snap.py +50 -4
  21. synapse/lib/storm.lark +4 -3
  22. synapse/lib/storm.py +96 -22
  23. synapse/lib/storm_format.py +1 -0
  24. synapse/lib/stormlib/aha.py +7 -1
  25. synapse/lib/stormlib/auth.py +13 -5
  26. synapse/lib/stormlib/cache.py +202 -0
  27. synapse/lib/stormlib/cortex.py +147 -8
  28. synapse/lib/stormlib/gen.py +53 -6
  29. synapse/lib/stormlib/math.py +1 -1
  30. synapse/lib/stormlib/model.py +11 -1
  31. synapse/lib/stormlib/spooled.py +109 -0
  32. synapse/lib/stormlib/vault.py +1 -1
  33. synapse/lib/stormtypes.py +113 -17
  34. synapse/lib/trigger.py +36 -47
  35. synapse/lib/types.py +29 -2
  36. synapse/lib/version.py +2 -2
  37. synapse/lib/view.py +80 -53
  38. synapse/models/economic.py +174 -5
  39. synapse/models/files.py +2 -0
  40. synapse/models/inet.py +77 -2
  41. synapse/models/infotech.py +12 -12
  42. synapse/models/orgs.py +72 -21
  43. synapse/models/person.py +40 -11
  44. synapse/models/risk.py +78 -24
  45. synapse/models/science.py +102 -0
  46. synapse/telepath.py +117 -35
  47. synapse/tests/test_cortex.py +84 -158
  48. synapse/tests/test_datamodel.py +22 -0
  49. synapse/tests/test_lib_agenda.py +52 -96
  50. synapse/tests/test_lib_aha.py +126 -4
  51. synapse/tests/test_lib_ast.py +412 -6
  52. synapse/tests/test_lib_cell.py +24 -8
  53. synapse/tests/test_lib_certdir.py +32 -0
  54. synapse/tests/test_lib_grammar.py +9 -1
  55. synapse/tests/test_lib_httpapi.py +0 -1
  56. synapse/tests/test_lib_jupyter.py +0 -1
  57. synapse/tests/test_lib_modelrev.py +41 -0
  58. synapse/tests/test_lib_nexus.py +38 -0
  59. synapse/tests/test_lib_storm.py +95 -5
  60. synapse/tests/test_lib_stormlib_cache.py +272 -0
  61. synapse/tests/test_lib_stormlib_cortex.py +71 -0
  62. synapse/tests/test_lib_stormlib_gen.py +37 -2
  63. synapse/tests/test_lib_stormlib_model.py +2 -0
  64. synapse/tests/test_lib_stormlib_spooled.py +190 -0
  65. synapse/tests/test_lib_stormlib_vault.py +12 -3
  66. synapse/tests/test_lib_stormsvc.py +0 -10
  67. synapse/tests/test_lib_stormtypes.py +60 -8
  68. synapse/tests/test_lib_trigger.py +20 -2
  69. synapse/tests/test_lib_types.py +17 -1
  70. synapse/tests/test_model_economic.py +114 -0
  71. synapse/tests/test_model_files.py +2 -0
  72. synapse/tests/test_model_inet.py +73 -1
  73. synapse/tests/test_model_infotech.py +2 -2
  74. synapse/tests/test_model_orgs.py +10 -1
  75. synapse/tests/test_model_risk.py +30 -2
  76. synapse/tests/test_model_science.py +59 -0
  77. synapse/tests/test_model_syn.py +0 -1
  78. synapse/tests/test_telepath.py +30 -7
  79. synapse/tests/test_tools_modrole.py +81 -0
  80. synapse/tests/test_tools_moduser.py +105 -0
  81. synapse/tools/modrole.py +59 -7
  82. synapse/tools/moduser.py +78 -10
  83. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/METADATA +2 -2
  84. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/RECORD +87 -83
  85. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/WHEEL +1 -1
  86. synapse/lib/provenance.py +0 -111
  87. synapse/tests/test_lib_provenance.py +0 -37
  88. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/LICENSE +0 -0
  89. {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/top_level.txt +0 -0
synapse/lib/cell.py CHANGED
@@ -79,6 +79,8 @@ permnames = {
79
79
  PERM_ADMIN: 'admin',
80
80
  }
81
81
 
82
+ diskspace = "Insufficient free space on disk."
83
+
82
84
  def adminapi(log=False):
83
85
  '''
84
86
  Decorator for CellApi (and subclasses) for requiring a method to be called only by an admin user.
@@ -121,7 +123,8 @@ async def _doIterBackup(path, chunksize=1024):
121
123
  link0, file1 = await s_link.linkfile()
122
124
 
123
125
  def dowrite(fd):
124
- with tarfile.open(output_filename, 'w|gz', fileobj=fd) as tar:
126
+ # TODO: When we are 3.12+ convert this back to w|gz - see https://github.com/python/cpython/pull/2962
127
+ with tarfile.open(output_filename, 'w:gz', fileobj=fd, compresslevel=1) as tar:
125
128
  tar.add(path, arcname=os.path.basename(path))
126
129
  fd.close()
127
130
 
@@ -341,7 +344,7 @@ class CellApi(s_base.Base):
341
344
  async def promote(self, graceful=False):
342
345
  return await self.cell.promote(graceful=graceful)
343
346
 
344
- @adminapi()
347
+ @adminapi(log=True)
345
348
  async def handoff(self, turl, timeout=30):
346
349
  return await self.cell.handoff(turl, timeout=timeout)
347
350
 
@@ -1057,6 +1060,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1057
1060
  self.cellparent = parent
1058
1061
  self.sessions = {}
1059
1062
  self.isactive = False
1063
+ self.activebase = None
1060
1064
  self.inaugural = False
1061
1065
  self.activecoros = {}
1062
1066
  self.sockaddr = None # Default value...
@@ -1155,6 +1159,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1155
1159
  self.usermetadb = self.slab.initdb('user:meta') # useriden + <valu> -> dict valu
1156
1160
  self.rolemetadb = self.slab.initdb('role:meta') # roleiden + <valu> -> dict valu
1157
1161
 
1162
+ # for runtime cell configuration values
1163
+ self.slab.initdb('cell:conf')
1164
+
1158
1165
  self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE)
1159
1166
 
1160
1167
  self.hive = await self._initCellHive()
@@ -1385,32 +1392,22 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1385
1392
 
1386
1393
  if (disk.free / disk.total) <= self.minfree:
1387
1394
 
1388
- reason = "Insufficient free space on disk."
1389
-
1390
- await self._setReadOnly(True, reason=reason)
1391
- nexsroot.setReadOnly(True, reason=reason)
1395
+ await nexsroot.addWriteHold(diskspace)
1392
1396
 
1393
1397
  mesg = f'Free space on {self.dirn} below minimum threshold (currently ' \
1394
1398
  f'{disk.free / disk.total * 100:.2f}%), setting Cell to read-only.'
1395
- logger.warning(mesg)
1399
+ logger.error(mesg)
1396
1400
 
1397
1401
  elif nexsroot.readonly:
1398
1402
 
1399
- await self._setReadOnly(False)
1400
- nexsroot.setReadOnly(False)
1401
-
1402
- await self.nexsroot.startup()
1403
+ await nexsroot.delWriteHold(diskspace)
1403
1404
 
1404
1405
  mesg = f'Free space on {self.dirn} above minimum threshold (currently ' \
1405
1406
  f'{disk.free / disk.total * 100:.2f}%), re-enabling writes.'
1406
- logger.warning(mesg)
1407
+ logger.error(mesg)
1407
1408
 
1408
1409
  await self._checkspace.timewait(timeout=self.FREE_SPACE_CHECK_FREQ)
1409
1410
 
1410
- async def _setReadOnly(self, valu, reason=None):
1411
- # implement any behavior necessary to change the cell read-only status
1412
- pass
1413
-
1414
1411
  def _getAhaAdmin(self):
1415
1412
  name = self.conf.get('aha:admin')
1416
1413
  if name is not None:
@@ -1563,32 +1560,22 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1563
1560
 
1564
1561
  self.ahasvcname = f'{ahaname}.{ahanetw}'
1565
1562
 
1566
- async def onlink(proxy):
1567
- while not proxy.isfini:
1568
- info = await self.getAhaInfo()
1563
+ async def _runAhaRegLoop():
1564
+
1565
+ while not self.isfini:
1569
1566
  try:
1567
+ proxy = await self.ahaclient.proxy()
1568
+ info = await self.getAhaInfo()
1570
1569
  await proxy.addAhaSvc(ahaname, info, network=ahanetw)
1571
1570
  if self.isactive and ahalead is not None:
1572
1571
  await proxy.addAhaSvc(ahalead, info, network=ahanetw)
1572
+ except Exception as e:
1573
+ logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}')
1574
+ await self.waitfini(1)
1575
+ else:
1576
+ await proxy.waitfini()
1573
1577
 
1574
- return
1575
-
1576
- except asyncio.CancelledError: # pragma: no cover
1577
- raise
1578
-
1579
- except Exception:
1580
- logger.exception('Error in _initAhaService() onlink')
1581
-
1582
- await proxy.waitfini(1)
1583
-
1584
- async def fini():
1585
- await self.ahaclient.offlink(onlink)
1586
-
1587
- async def init():
1588
- await self.ahaclient.onlink(onlink)
1589
- self.onfini(fini)
1590
-
1591
- self.schedCoro(init())
1578
+ self.schedCoro(_runAhaRegLoop())
1592
1579
 
1593
1580
  async def initServiceRuntime(self):
1594
1581
  pass
@@ -1675,9 +1662,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1675
1662
  mesg = 'promote() called on non-mirror'
1676
1663
  raise s_exc.BadConfValu(mesg=mesg)
1677
1664
 
1665
+ ahaname = self.conf.get('aha:name')
1666
+ logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful} ahaname={ahaname}')
1667
+
1678
1668
  if graceful:
1679
1669
 
1680
- ahaname = self.conf.get('aha:name')
1681
1670
  if ahaname is None: # pragma: no cover
1682
1671
  mesg = 'Cannot gracefully promote without aha:name configured.'
1683
1672
  raise s_exc.BadArg(mesg=mesg)
@@ -1688,21 +1677,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1688
1677
  raise s_exc.BadArg(mesg=mesg)
1689
1678
 
1690
1679
  myurl = f'aha://{ahaname}.{ahanetw}'
1680
+ logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}')
1691
1681
  async with await s_telepath.openurl(mirurl) as lead:
1682
+ logger.debug(f'PROMOTION: Requesting leadership handoff to ahaname={ahaname}')
1692
1683
  await lead.handoff(myurl)
1684
+ logger.warning(f'PROMOTION: Completed leadership handoff to ahaname={ahaname}')
1693
1685
  return
1694
1686
 
1687
+ logger.debug(f'PROMOTION: Clearing mirror configuration for ahaname={ahaname}')
1695
1688
  self.modCellConf({'mirror': None})
1696
1689
 
1690
+ logger.debug(f'PROMOTION: Promoting the nexus root for ahaname={ahaname}')
1697
1691
  await self.nexsroot.promote()
1692
+
1693
+ logger.debug(f'PROMOTION: Setting the cell as active ahaname={ahaname}')
1698
1694
  await self.setCellActive(True)
1699
1695
 
1696
+ logger.warning(f'PROMOTION: Finished leadership promotion ahaname={ahaname}')
1697
+
1700
1698
  async def handoff(self, turl, timeout=30):
1701
1699
  '''
1702
1700
  Hand off leadership to a mirror in a transactional fashion.
1703
1701
  '''
1702
+ ahaname = self.conf.get("aha:name")
1703
+ logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
1704
1704
  async with await s_telepath.openurl(turl) as cell:
1705
1705
 
1706
+ logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
1707
+
1706
1708
  if self.iden != await cell.getCellIden(): # pragma: no cover
1707
1709
  mesg = 'Mirror handoff remote cell iden does not match!'
1708
1710
  raise s_exc.BadArg(mesg=mesg)
@@ -1711,20 +1713,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1711
1713
  mesg = 'Cannot handoff mirror leadership to myself!'
1712
1714
  raise s_exc.BadArg(mesg=mesg)
1713
1715
 
1716
+ logger.debug(f'HANDOFF: Obtaining nexus applylock ahaname={ahaname}')
1717
+
1714
1718
  async with self.nexsroot.applylock:
1715
1719
 
1720
+ logger.debug(f'HANDOFF: Obtained nexus applylock ahaname={ahaname}')
1716
1721
  indx = await self.getNexsIndx()
1722
+
1723
+ logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}')
1717
1724
  if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover
1718
1725
  mndx = await cell.getNexsIndx()
1719
1726
  mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.'
1720
1727
  raise s_exc.NotReady(mesg=mesg)
1721
1728
 
1729
+ logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion ahaname={ahaname}')
1722
1730
  await cell.promote()
1731
+
1732
+ logger.debug(f'HANDOFF: Setting the service as inactive ahaname={ahaname}')
1723
1733
  await self.setCellActive(False)
1724
1734
 
1735
+ logger.debug(f'HANDOFF: Configuring service to use the new leader as its mirror ahaname={ahaname}')
1725
1736
  self.modCellConf({'mirror': turl})
1737
+
1738
+ logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}')
1726
1739
  await self.nexsroot.startup()
1727
1740
 
1741
+ logger.debug(f'HANDOFF: Released nexus applylock ahaname={ahaname}')
1742
+ logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}')
1743
+
1728
1744
  async def reqAhaProxy(self, timeout=None):
1729
1745
  if self.ahaclient is None:
1730
1746
  mesg = 'AHA is not configured on this service.'
@@ -1855,18 +1871,32 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
1855
1871
  return self.isactive
1856
1872
 
1857
1873
  async def setCellActive(self, active):
1874
+
1875
+ if active == self.isactive:
1876
+ return
1877
+
1858
1878
  self.isactive = active
1859
1879
 
1860
1880
  if self.isactive:
1881
+ self.activebase = await s_base.Base.anit()
1882
+ self.onfini(self.activebase)
1861
1883
  self._fireActiveCoros()
1862
1884
  await self._execCellUpdates()
1863
1885
  await self.initServiceActive()
1864
1886
  else:
1865
1887
  await self._killActiveCoros()
1888
+ await self.activebase.fini()
1889
+ self.activebase = None
1866
1890
  await self.initServicePassive()
1867
1891
 
1868
1892
  await self._setAhaActive()
1869
1893
 
1894
+ def runActiveTask(self, coro):
1895
+ # an API for active coroutines to use when running an
1896
+ # ephemeral task which should be automatically torn down
1897
+ # if the cell becomes inactive
1898
+ return self.activebase.schedCoro(coro)
1899
+
1870
1900
  async def initServiceActive(self): # pragma: no cover
1871
1901
  pass
1872
1902
 
@@ -3654,7 +3684,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
3654
3684
 
3655
3685
  await self.ahaclient.waitready()
3656
3686
 
3657
- mirrors = await self.ahaclient.getAhaSvcMirrors(self.ahasvcname)
3687
+ proxy = await self.ahaclient.proxy(timeout=5)
3688
+ mirrors = await proxy.getAhaSvcMirrors(self.ahasvcname)
3658
3689
  if mirrors is None:
3659
3690
  mesg = 'Service must be configured with AHA to enumerate mirror URLs'
3660
3691
  raise s_exc.NoSuchName(mesg=mesg, name=self.ahasvcname)
synapse/lib/certdir.py CHANGED
@@ -1501,6 +1501,11 @@ class CertDir:
1501
1501
  return c_rsa.generate_private_key(65537, self.crypto_numbits)
1502
1502
 
1503
1503
  def _genCertBuilder(self, name: str, pubkey: c_types.PublicKeyTypes) -> c_x509.CertificateBuilder:
1504
+
1505
+ if not 1 <= len(name) <= 64:
1506
+ mesg = f'Certificate name values must be between 1-64 characters. got name={name}, len={len(name)}'
1507
+ raise s_exc.CryptoErr(mesg=mesg)
1508
+
1504
1509
  builder = c_x509.CertificateBuilder()
1505
1510
  builder = builder.subject_name(c_x509.Name([
1506
1511
  c_x509.NameAttribute(c_x509.NameOID.COMMON_NAME, name),
@@ -1515,6 +1520,10 @@ class CertDir:
1515
1520
 
1516
1521
  def _genPkeyCsr(self, name: str, mode: str, outp: OutPutOrNone = None) -> bytes:
1517
1522
 
1523
+ if not 1 <= len(name) <= 64:
1524
+ mesg = f'CSR name values must be between 1-64 characters. got name={name}, len={len(name)}'
1525
+ raise s_exc.CryptoErr(mesg=mesg)
1526
+
1518
1527
  pkey = self._genPrivKey()
1519
1528
 
1520
1529
  builder = c_x509.CertificateSigningRequestBuilder()
synapse/lib/hiveauth.py CHANGED
@@ -1,4 +1,7 @@
1
1
  import logging
2
+ import dataclasses
3
+
4
+ from typing import Union
2
5
 
3
6
  import synapse.exc as s_exc
4
7
  import synapse.common as s_common
@@ -38,6 +41,45 @@ def textFromRule(rule):
38
41
  text = '!' + text
39
42
  return text
40
43
 
44
+ @dataclasses.dataclass(slots=True)
45
+ class _allowedReason:
46
+ value: Union[bool | None]
47
+ default: bool = False
48
+ isadmin: bool = False
49
+ islocked: bool = False
50
+ gateiden: Union[str | None] = None
51
+ roleiden: Union[str | None] = None
52
+ rolename: Union[str | None] = None
53
+ rule: tuple = ()
54
+
55
+ @property
56
+ def mesg(self):
57
+ if self.islocked:
58
+ return 'The user is locked.'
59
+ if self.default:
60
+ return 'No matching rule found.'
61
+
62
+ if self.isadmin:
63
+ if self.gateiden:
64
+ return f'The user is an admin of auth gate {self.gateiden}.'
65
+ return 'The user is a global admin.'
66
+
67
+ if self.rule:
68
+ rt = textFromRule((self.value, self.rule))
69
+ if self.gateiden:
70
+ if self.roleiden:
71
+ m = f'Matched role rule ({rt}) for role {self.rolename} on gate {self.gateiden}.'
72
+ else:
73
+ m = f'Matched user rule ({rt}) on gate {self.gateiden}.'
74
+ else:
75
+ if self.roleiden:
76
+ m = f'Matched role rule ({rt}) for role {self.rolename}.'
77
+ else:
78
+ m = f'Matched user rule ({rt}).'
79
+ return m
80
+
81
+ return 'No matching rule found.'
82
+
41
83
  class Auth(s_nexus.Pusher):
42
84
  '''
43
85
  Auth is a user authentication and authorization stored in a Hive. Users
@@ -827,7 +869,7 @@ class HiveRole(HiveRuler):
827
869
  return allow
828
870
  return default
829
871
 
830
- # 2. check user rules
872
+ # 2. check role rules
831
873
  for allow, path in self.info.get('rules', ()):
832
874
  if perm[:len(path)] == path:
833
875
  return allow
@@ -858,6 +900,7 @@ class HiveUser(HiveRuler):
858
900
  self.vars = await varz.dict(nexs=True)
859
901
 
860
902
  self.permcache = s_cache.FixedCache(self._allowed)
903
+ self.allowedcache = s_cache.FixedCache(self._getAllowedReason)
861
904
 
862
905
  def pack(self, packroles=False):
863
906
 
@@ -903,6 +946,9 @@ class HiveUser(HiveRuler):
903
946
  return self.permcache.get((perm, default, gateiden))
904
947
 
905
948
  def _allowed(self, pkey):
949
+ '''
950
+ NOTE: This must remain in sync with any changes to _getAllowedReason()!
951
+ '''
906
952
 
907
953
  perm, default, gateiden = pkey
908
954
 
@@ -951,18 +997,23 @@ class HiveUser(HiveRuler):
951
997
 
952
998
  return default
953
999
 
954
- def getAllowedReason(self, perm, gateiden=None, default=False):
1000
+ def getAllowedReason(self, perm, default=None, gateiden=None):
1001
+ '''
1002
+ A routine which will return a tuple of (allowed, info).
955
1003
  '''
956
- A non-optimized diagnostic routine which will return a tuple
957
- of (allowed, reason). This is implemented separately for perf.
1004
+ perm = tuple(perm)
1005
+ return self.allowedcache.get((perm, default, gateiden))
958
1006
 
1007
+ def _getAllowedReason(self, pkey):
1008
+ '''
959
1009
  NOTE: This must remain in sync with any changes to _allowed()!
960
1010
  '''
1011
+ perm, default, gateiden = pkey
961
1012
  if self.info.get('locked'):
962
- return (False, 'The user is locked.')
1013
+ return _allowedReason(False, islocked=True)
963
1014
 
964
1015
  if self.info.get('admin'):
965
- return (True, 'The user is a global admin.')
1016
+ return _allowedReason(True, isadmin=True)
966
1017
 
967
1018
  # 1. check authgate user rules
968
1019
  if gateiden is not None:
@@ -971,16 +1022,16 @@ class HiveUser(HiveRuler):
971
1022
  if info is not None:
972
1023
 
973
1024
  if info.get('admin'):
974
- return (True, f'The user is an admin of auth gate {gateiden}.')
1025
+ return _allowedReason(True, isadmin=True, gateiden=gateiden)
975
1026
 
976
1027
  for allow, path in info.get('rules', ()):
977
1028
  if perm[:len(path)] == path:
978
- return (allow, f'Matched user rule ({textFromRule((allow, path))}) on gate {gateiden}.')
1029
+ return _allowedReason(allow, gateiden=gateiden, rule=path)
979
1030
 
980
1031
  # 2. check user rules
981
1032
  for allow, path in self.info.get('rules', ()):
982
1033
  if perm[:len(path)] == path:
983
- return (allow, f'Matched user rule ({textFromRule((allow, path))}).')
1034
+ return _allowedReason(allow, rule=path)
984
1035
 
985
1036
  # 3. check authgate role rules
986
1037
  if gateiden is not None:
@@ -993,18 +1044,20 @@ class HiveUser(HiveRuler):
993
1044
 
994
1045
  for allow, path in info.get('rules', ()):
995
1046
  if perm[:len(path)] == path:
996
- return (allow, f'Matched role rule ({textFromRule((allow, path))}) for role {role.name} on gate {gateiden}.')
1047
+ return _allowedReason(allow, gateiden=gateiden, roleiden=role.iden, rolename=role.name,
1048
+ rule=path)
997
1049
 
998
1050
  # 4. check role rules
999
1051
  for role in self.getRoles():
1000
1052
  for allow, path in role.info.get('rules', ()):
1001
1053
  if perm[:len(path)] == path:
1002
- return (allow, f'Matched role rule ({textFromRule((allow, path))}) for role {role.name}.')
1054
+ return _allowedReason(allow, roleiden=role.iden, rolename=role.name, rule=path)
1003
1055
 
1004
- return (default, 'No matching rule found.')
1056
+ return _allowedReason(default, default=True)
1005
1057
 
1006
1058
  def clearAuthCache(self):
1007
1059
  self.permcache.clear()
1060
+ self.allowedcache.clear()
1008
1061
 
1009
1062
  async def genGateInfo(self, gateiden):
1010
1063
  info = self.authgates.get(gateiden)
synapse/lib/httpapi.py CHANGED
@@ -1344,6 +1344,7 @@ class ExtApiHandler(StormHandler):
1344
1344
  varz['_http_request_info'] = info
1345
1345
 
1346
1346
  opts = {
1347
+ 'mirror': adef.get('pool', False),
1347
1348
  'readonly': adef.get('readonly'),
1348
1349
  'show': (
1349
1350
  'http:resp:body',
synapse/lib/modelrev.py CHANGED
@@ -8,7 +8,7 @@ import synapse.lib.layer as s_layer
8
8
 
9
9
  logger = logging.getLogger(__name__)
10
10
 
11
- maxvers = (0, 2, 23)
11
+ maxvers = (0, 2, 24)
12
12
 
13
13
  class ModelRev:
14
14
 
@@ -37,6 +37,7 @@ class ModelRev:
37
37
  ((0, 2, 21), self.revModel_0_2_21),
38
38
  ((0, 2, 22), self.revModel_0_2_22),
39
39
  ((0, 2, 23), self.revModel_0_2_23),
40
+ ((0, 2, 24), self.revModel_0_2_24),
40
41
  )
41
42
 
42
43
  async def _uniqSortArray(self, todoprops, layers):
@@ -733,6 +734,29 @@ class ModelRev:
733
734
  async def revModel_0_2_23(self, layers):
734
735
  await self._normFormSubs(layers, 'inet:ipv6')
735
736
 
737
+ async def revModel_0_2_24(self, layers):
738
+ await self._normPropValu(layers, 'risk:mitigation:name')
739
+ await self._normPropValu(layers, 'it:mitre:attack:technique:name')
740
+ await self._normPropValu(layers, 'it:mitre:attack:mitigation:name')
741
+
742
+ formprops = {}
743
+ for prop in self.core.model.getPropsByType('velocity'):
744
+ formname = prop.form.name
745
+ if formname not in formprops:
746
+ formprops[formname] = []
747
+
748
+ formprops[formname].append(prop)
749
+
750
+ for prop in self.core.model.getArrayPropsByType('velocity'):
751
+ formname = prop.form.name
752
+ if formname not in formprops:
753
+ formprops[formname] = []
754
+
755
+ formprops[formname].append(prop)
756
+
757
+ for form, props in formprops.items():
758
+ await self._normVelocityProps(layers, form, props)
759
+
736
760
  async def runStorm(self, text, opts=None):
737
761
  '''
738
762
  Run storm code in a schedcoro and log the output messages.
@@ -759,52 +783,50 @@ class ModelRev:
759
783
 
760
784
  async def revCoreLayers(self):
761
785
 
762
- async with self.core.enterMigrationMode():
763
-
764
- version = self.revs[-1][0] if self.revs else maxvers
786
+ version = self.revs[-1][0] if self.revs else maxvers
765
787
 
766
- # do a first pass to detect layers at the wrong version
767
- # that we are not able to rev ourselves and bail...
788
+ # do a first pass to detect layers at the wrong version
789
+ # that we are not able to rev ourselves and bail...
768
790
 
769
- layers = []
770
- for layr in self.core.layers.values():
791
+ layers = []
792
+ for layr in self.core.layers.values():
771
793
 
772
- if layr.fresh:
773
- await layr.setModelVers(version)
774
- continue
794
+ if layr.fresh:
795
+ await layr.setModelVers(version)
796
+ continue
775
797
 
776
- vers = await layr.getModelVers()
777
- if vers == version:
778
- continue
798
+ vers = await layr.getModelVers()
799
+ if vers == version:
800
+ continue
779
801
 
780
- if not layr.canrev and vers != version:
781
- mesg = f'layer {layr.__class__.__name__} {layr.iden} ({layr.dirn}) can not be updated.'
782
- raise s_exc.CantRevLayer(layer=layr.iden, mesg=mesg, curv=version, layv=vers)
802
+ if not layr.canrev and vers != version:
803
+ mesg = f'layer {layr.__class__.__name__} {layr.iden} ({layr.dirn}) can not be updated.'
804
+ raise s_exc.CantRevLayer(layer=layr.iden, mesg=mesg, curv=version, layv=vers)
783
805
 
784
- if vers > version:
785
- mesg = f'layer {layr.__class__.__name__} {layr.iden} ({layr.dirn}) is from the future!'
786
- raise s_exc.CantRevLayer(layer=layr.iden, mesg=mesg, curv=version, layv=vers)
806
+ if vers > version:
807
+ mesg = f'layer {layr.__class__.__name__} {layr.iden} ({layr.dirn}) is from the future!'
808
+ raise s_exc.CantRevLayer(layer=layr.iden, mesg=mesg, curv=version, layv=vers)
787
809
 
788
- # realistically all layers are probably at the same version... but...
789
- layers.append(layr)
810
+ # realistically all layers are probably at the same version... but...
811
+ layers.append(layr)
790
812
 
791
- # got anything to do?
792
- if not layers:
793
- return
813
+ # got anything to do?
814
+ if not layers:
815
+ return
794
816
 
795
- for revvers, revmeth in self.revs:
817
+ for revvers, revmeth in self.revs:
796
818
 
797
- todo = [lyr for lyr in layers if not lyr.ismirror and await lyr.getModelVers() < revvers]
798
- if not todo:
799
- continue
819
+ todo = [lyr for lyr in layers if not lyr.ismirror and await lyr.getModelVers() < revvers]
820
+ if not todo:
821
+ continue
800
822
 
801
- logger.warning(f'beginning model migration -> {revvers}')
823
+ logger.warning(f'beginning model migration -> {revvers}')
802
824
 
803
- await revmeth(todo)
825
+ await revmeth(todo)
804
826
 
805
- [await lyr.setModelVers(revvers) for lyr in todo]
827
+ [await lyr.setModelVers(revvers) for lyr in todo]
806
828
 
807
- logger.warning('...model migrations complete!')
829
+ logger.warning('...model migrations complete!')
808
830
 
809
831
  async def _normPropValu(self, layers, propfull):
810
832
 
@@ -850,6 +872,72 @@ class ModelRev:
850
872
  if nodeedits:
851
873
  await save()
852
874
 
875
+ async def _normVelocityProps(self, layers, form, props):
876
+
877
+ meta = {'time': s_common.now(), 'user': self.core.auth.rootuser.iden}
878
+
879
+ nodeedits = []
880
+ for layr in layers:
881
+
882
+ async def save():
883
+ await layr.storNodeEdits(nodeedits, meta)
884
+ nodeedits.clear()
885
+
886
+ async for buid, formvalu in layr.iterFormRows(form):
887
+ sode = layr._getStorNode(buid)
888
+ if (nodeprops := sode.get('props')) is None:
889
+ continue
890
+
891
+ for prop in props:
892
+ if (curv := nodeprops.get(prop.name)) is None:
893
+ continue
894
+
895
+ propvalu = curv[0]
896
+ if prop.type.isarray:
897
+ hasfloat = False
898
+ strvalu = []
899
+ for valu in propvalu:
900
+ if isinstance(valu, float):
901
+ strvalu.append(str(valu))
902
+ hasfloat = True
903
+
904
+ if not hasfloat:
905
+ continue
906
+ else:
907
+ if not isinstance(propvalu, float):
908
+ continue
909
+ strvalu = str(propvalu)
910
+
911
+ nodeprops.pop(prop.name)
912
+
913
+ try:
914
+ norm, info = prop.type.norm(strvalu)
915
+ except s_exc.BadTypeValu as e:
916
+ nodeedits.append(
917
+ (buid, form, (
918
+ (s_layer.EDIT_NODEDATA_SET, (f'_migrated:{prop.full}', propvalu, None), ()),
919
+ (s_layer.EDIT_PROP_DEL, (prop.name, propvalu, prop.type.stortype), ()),
920
+ )),
921
+ )
922
+
923
+ oldm = e.errinfo.get('mesg')
924
+ iden = s_common.ehex(buid)
925
+ logger.warning(f'error re-norming {prop.full}={propvalu} (layer: {layr.iden}, node: {iden}): {oldm}',
926
+ extra={'synapse': {'node': iden, 'layer': layr.iden}})
927
+ continue
928
+
929
+ nodeedits.append(
930
+ (buid, form, (
931
+ (s_layer.EDIT_PROP_SET, (prop.name, norm, propvalu, prop.type.stortype), ()),
932
+ )),
933
+ )
934
+
935
+ if len(nodeedits) >= 1000: # pragma: no cover
936
+ await save()
937
+
938
+ if nodeedits:
939
+ await save()
940
+
853
941
  async def _updatePropStortype(self, layers, propfull):
854
942
 
855
943
  meta = {'time': s_common.now(), 'user': self.core.auth.rootuser.iden}
synapse/lib/modules.py CHANGED
@@ -27,4 +27,5 @@ coremods = (
27
27
  'synapse.models.proj.ProjectModule',
28
28
  'synapse.models.biz.BizModule',
29
29
  'synapse.models.belief.BeliefModule',
30
+ 'synapse.models.science.ScienceModule',
30
31
  )