synapse 2.164.0__py311-none-any.whl → 2.166.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +3 -3
- synapse/cmds/cortex.py +1 -6
- synapse/common.py +7 -1
- synapse/cortex.py +145 -192
- synapse/datamodel.py +36 -1
- synapse/lib/agenda.py +87 -97
- synapse/lib/aha.py +51 -0
- synapse/lib/ast.py +22 -23
- synapse/lib/base.py +0 -6
- synapse/lib/boss.py +3 -0
- synapse/lib/cell.py +70 -39
- synapse/lib/certdir.py +9 -0
- synapse/lib/hiveauth.py +65 -12
- synapse/lib/httpapi.py +1 -0
- synapse/lib/modelrev.py +121 -33
- synapse/lib/modules.py +1 -0
- synapse/lib/nexus.py +64 -26
- synapse/lib/parser.py +2 -0
- synapse/lib/schemas.py +14 -0
- synapse/lib/snap.py +50 -4
- synapse/lib/storm.lark +4 -3
- synapse/lib/storm.py +96 -22
- synapse/lib/storm_format.py +1 -0
- synapse/lib/stormlib/aha.py +7 -1
- synapse/lib/stormlib/auth.py +13 -5
- synapse/lib/stormlib/cache.py +202 -0
- synapse/lib/stormlib/cortex.py +147 -8
- synapse/lib/stormlib/gen.py +53 -6
- synapse/lib/stormlib/math.py +1 -1
- synapse/lib/stormlib/model.py +11 -1
- synapse/lib/stormlib/spooled.py +109 -0
- synapse/lib/stormlib/vault.py +1 -1
- synapse/lib/stormtypes.py +113 -17
- synapse/lib/trigger.py +36 -47
- synapse/lib/types.py +29 -2
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +80 -53
- synapse/models/economic.py +174 -5
- synapse/models/files.py +2 -0
- synapse/models/inet.py +77 -2
- synapse/models/infotech.py +12 -12
- synapse/models/orgs.py +72 -21
- synapse/models/person.py +40 -11
- synapse/models/risk.py +78 -24
- synapse/models/science.py +102 -0
- synapse/telepath.py +117 -35
- synapse/tests/test_cortex.py +84 -158
- synapse/tests/test_datamodel.py +22 -0
- synapse/tests/test_lib_agenda.py +52 -96
- synapse/tests/test_lib_aha.py +126 -4
- synapse/tests/test_lib_ast.py +412 -6
- synapse/tests/test_lib_cell.py +24 -8
- synapse/tests/test_lib_certdir.py +32 -0
- synapse/tests/test_lib_grammar.py +9 -1
- synapse/tests/test_lib_httpapi.py +0 -1
- synapse/tests/test_lib_jupyter.py +0 -1
- synapse/tests/test_lib_modelrev.py +41 -0
- synapse/tests/test_lib_nexus.py +38 -0
- synapse/tests/test_lib_storm.py +95 -5
- synapse/tests/test_lib_stormlib_cache.py +272 -0
- synapse/tests/test_lib_stormlib_cortex.py +71 -0
- synapse/tests/test_lib_stormlib_gen.py +37 -2
- synapse/tests/test_lib_stormlib_model.py +2 -0
- synapse/tests/test_lib_stormlib_spooled.py +190 -0
- synapse/tests/test_lib_stormlib_vault.py +12 -3
- synapse/tests/test_lib_stormsvc.py +0 -10
- synapse/tests/test_lib_stormtypes.py +60 -8
- synapse/tests/test_lib_trigger.py +20 -2
- synapse/tests/test_lib_types.py +17 -1
- synapse/tests/test_model_economic.py +114 -0
- synapse/tests/test_model_files.py +2 -0
- synapse/tests/test_model_inet.py +73 -1
- synapse/tests/test_model_infotech.py +2 -2
- synapse/tests/test_model_orgs.py +10 -1
- synapse/tests/test_model_risk.py +30 -2
- synapse/tests/test_model_science.py +59 -0
- synapse/tests/test_model_syn.py +0 -1
- synapse/tests/test_telepath.py +30 -7
- synapse/tests/test_tools_modrole.py +81 -0
- synapse/tests/test_tools_moduser.py +105 -0
- synapse/tools/modrole.py +59 -7
- synapse/tools/moduser.py +78 -10
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/METADATA +2 -2
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/RECORD +87 -83
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/WHEEL +1 -1
- synapse/lib/provenance.py +0 -111
- synapse/tests/test_lib_provenance.py +0 -37
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/LICENSE +0 -0
- {synapse-2.164.0.dist-info → synapse-2.166.0.dist-info}/top_level.txt +0 -0
synapse/lib/cell.py
CHANGED
|
@@ -79,6 +79,8 @@ permnames = {
|
|
|
79
79
|
PERM_ADMIN: 'admin',
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
diskspace = "Insufficient free space on disk."
|
|
83
|
+
|
|
82
84
|
def adminapi(log=False):
|
|
83
85
|
'''
|
|
84
86
|
Decorator for CellApi (and subclasses) for requiring a method to be called only by an admin user.
|
|
@@ -121,7 +123,8 @@ async def _doIterBackup(path, chunksize=1024):
|
|
|
121
123
|
link0, file1 = await s_link.linkfile()
|
|
122
124
|
|
|
123
125
|
def dowrite(fd):
|
|
124
|
-
|
|
126
|
+
# TODO: When we are 3.12+ convert this back to w|gz - see https://github.com/python/cpython/pull/2962
|
|
127
|
+
with tarfile.open(output_filename, 'w:gz', fileobj=fd, compresslevel=1) as tar:
|
|
125
128
|
tar.add(path, arcname=os.path.basename(path))
|
|
126
129
|
fd.close()
|
|
127
130
|
|
|
@@ -341,7 +344,7 @@ class CellApi(s_base.Base):
|
|
|
341
344
|
async def promote(self, graceful=False):
|
|
342
345
|
return await self.cell.promote(graceful=graceful)
|
|
343
346
|
|
|
344
|
-
@adminapi()
|
|
347
|
+
@adminapi(log=True)
|
|
345
348
|
async def handoff(self, turl, timeout=30):
|
|
346
349
|
return await self.cell.handoff(turl, timeout=timeout)
|
|
347
350
|
|
|
@@ -1057,6 +1060,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1057
1060
|
self.cellparent = parent
|
|
1058
1061
|
self.sessions = {}
|
|
1059
1062
|
self.isactive = False
|
|
1063
|
+
self.activebase = None
|
|
1060
1064
|
self.inaugural = False
|
|
1061
1065
|
self.activecoros = {}
|
|
1062
1066
|
self.sockaddr = None # Default value...
|
|
@@ -1155,6 +1159,9 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1155
1159
|
self.usermetadb = self.slab.initdb('user:meta') # useriden + <valu> -> dict valu
|
|
1156
1160
|
self.rolemetadb = self.slab.initdb('role:meta') # roleiden + <valu> -> dict valu
|
|
1157
1161
|
|
|
1162
|
+
# for runtime cell configuration values
|
|
1163
|
+
self.slab.initdb('cell:conf')
|
|
1164
|
+
|
|
1158
1165
|
self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE)
|
|
1159
1166
|
|
|
1160
1167
|
self.hive = await self._initCellHive()
|
|
@@ -1385,32 +1392,22 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1385
1392
|
|
|
1386
1393
|
if (disk.free / disk.total) <= self.minfree:
|
|
1387
1394
|
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
await self._setReadOnly(True, reason=reason)
|
|
1391
|
-
nexsroot.setReadOnly(True, reason=reason)
|
|
1395
|
+
await nexsroot.addWriteHold(diskspace)
|
|
1392
1396
|
|
|
1393
1397
|
mesg = f'Free space on {self.dirn} below minimum threshold (currently ' \
|
|
1394
1398
|
f'{disk.free / disk.total * 100:.2f}%), setting Cell to read-only.'
|
|
1395
|
-
logger.
|
|
1399
|
+
logger.error(mesg)
|
|
1396
1400
|
|
|
1397
1401
|
elif nexsroot.readonly:
|
|
1398
1402
|
|
|
1399
|
-
await
|
|
1400
|
-
nexsroot.setReadOnly(False)
|
|
1401
|
-
|
|
1402
|
-
await self.nexsroot.startup()
|
|
1403
|
+
await nexsroot.delWriteHold(diskspace)
|
|
1403
1404
|
|
|
1404
1405
|
mesg = f'Free space on {self.dirn} above minimum threshold (currently ' \
|
|
1405
1406
|
f'{disk.free / disk.total * 100:.2f}%), re-enabling writes.'
|
|
1406
|
-
logger.
|
|
1407
|
+
logger.error(mesg)
|
|
1407
1408
|
|
|
1408
1409
|
await self._checkspace.timewait(timeout=self.FREE_SPACE_CHECK_FREQ)
|
|
1409
1410
|
|
|
1410
|
-
async def _setReadOnly(self, valu, reason=None):
|
|
1411
|
-
# implement any behavior necessary to change the cell read-only status
|
|
1412
|
-
pass
|
|
1413
|
-
|
|
1414
1411
|
def _getAhaAdmin(self):
|
|
1415
1412
|
name = self.conf.get('aha:admin')
|
|
1416
1413
|
if name is not None:
|
|
@@ -1563,32 +1560,22 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1563
1560
|
|
|
1564
1561
|
self.ahasvcname = f'{ahaname}.{ahanetw}'
|
|
1565
1562
|
|
|
1566
|
-
async def
|
|
1567
|
-
|
|
1568
|
-
|
|
1563
|
+
async def _runAhaRegLoop():
|
|
1564
|
+
|
|
1565
|
+
while not self.isfini:
|
|
1569
1566
|
try:
|
|
1567
|
+
proxy = await self.ahaclient.proxy()
|
|
1568
|
+
info = await self.getAhaInfo()
|
|
1570
1569
|
await proxy.addAhaSvc(ahaname, info, network=ahanetw)
|
|
1571
1570
|
if self.isactive and ahalead is not None:
|
|
1572
1571
|
await proxy.addAhaSvc(ahalead, info, network=ahanetw)
|
|
1572
|
+
except Exception as e:
|
|
1573
|
+
logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}')
|
|
1574
|
+
await self.waitfini(1)
|
|
1575
|
+
else:
|
|
1576
|
+
await proxy.waitfini()
|
|
1573
1577
|
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
except asyncio.CancelledError: # pragma: no cover
|
|
1577
|
-
raise
|
|
1578
|
-
|
|
1579
|
-
except Exception:
|
|
1580
|
-
logger.exception('Error in _initAhaService() onlink')
|
|
1581
|
-
|
|
1582
|
-
await proxy.waitfini(1)
|
|
1583
|
-
|
|
1584
|
-
async def fini():
|
|
1585
|
-
await self.ahaclient.offlink(onlink)
|
|
1586
|
-
|
|
1587
|
-
async def init():
|
|
1588
|
-
await self.ahaclient.onlink(onlink)
|
|
1589
|
-
self.onfini(fini)
|
|
1590
|
-
|
|
1591
|
-
self.schedCoro(init())
|
|
1578
|
+
self.schedCoro(_runAhaRegLoop())
|
|
1592
1579
|
|
|
1593
1580
|
async def initServiceRuntime(self):
|
|
1594
1581
|
pass
|
|
@@ -1675,9 +1662,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1675
1662
|
mesg = 'promote() called on non-mirror'
|
|
1676
1663
|
raise s_exc.BadConfValu(mesg=mesg)
|
|
1677
1664
|
|
|
1665
|
+
ahaname = self.conf.get('aha:name')
|
|
1666
|
+
logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful} ahaname={ahaname}')
|
|
1667
|
+
|
|
1678
1668
|
if graceful:
|
|
1679
1669
|
|
|
1680
|
-
ahaname = self.conf.get('aha:name')
|
|
1681
1670
|
if ahaname is None: # pragma: no cover
|
|
1682
1671
|
mesg = 'Cannot gracefully promote without aha:name configured.'
|
|
1683
1672
|
raise s_exc.BadArg(mesg=mesg)
|
|
@@ -1688,21 +1677,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1688
1677
|
raise s_exc.BadArg(mesg=mesg)
|
|
1689
1678
|
|
|
1690
1679
|
myurl = f'aha://{ahaname}.{ahanetw}'
|
|
1680
|
+
logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}')
|
|
1691
1681
|
async with await s_telepath.openurl(mirurl) as lead:
|
|
1682
|
+
logger.debug(f'PROMOTION: Requesting leadership handoff to ahaname={ahaname}')
|
|
1692
1683
|
await lead.handoff(myurl)
|
|
1684
|
+
logger.warning(f'PROMOTION: Completed leadership handoff to ahaname={ahaname}')
|
|
1693
1685
|
return
|
|
1694
1686
|
|
|
1687
|
+
logger.debug(f'PROMOTION: Clearing mirror configuration for ahaname={ahaname}')
|
|
1695
1688
|
self.modCellConf({'mirror': None})
|
|
1696
1689
|
|
|
1690
|
+
logger.debug(f'PROMOTION: Promoting the nexus root for ahaname={ahaname}')
|
|
1697
1691
|
await self.nexsroot.promote()
|
|
1692
|
+
|
|
1693
|
+
logger.debug(f'PROMOTION: Setting the cell as active ahaname={ahaname}')
|
|
1698
1694
|
await self.setCellActive(True)
|
|
1699
1695
|
|
|
1696
|
+
logger.warning(f'PROMOTION: Finished leadership promotion ahaname={ahaname}')
|
|
1697
|
+
|
|
1700
1698
|
async def handoff(self, turl, timeout=30):
|
|
1701
1699
|
'''
|
|
1702
1700
|
Hand off leadership to a mirror in a transactional fashion.
|
|
1703
1701
|
'''
|
|
1702
|
+
ahaname = self.conf.get("aha:name")
|
|
1703
|
+
logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
|
|
1704
1704
|
async with await s_telepath.openurl(turl) as cell:
|
|
1705
1705
|
|
|
1706
|
+
logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
|
|
1707
|
+
|
|
1706
1708
|
if self.iden != await cell.getCellIden(): # pragma: no cover
|
|
1707
1709
|
mesg = 'Mirror handoff remote cell iden does not match!'
|
|
1708
1710
|
raise s_exc.BadArg(mesg=mesg)
|
|
@@ -1711,20 +1713,34 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1711
1713
|
mesg = 'Cannot handoff mirror leadership to myself!'
|
|
1712
1714
|
raise s_exc.BadArg(mesg=mesg)
|
|
1713
1715
|
|
|
1716
|
+
logger.debug(f'HANDOFF: Obtaining nexus applylock ahaname={ahaname}')
|
|
1717
|
+
|
|
1714
1718
|
async with self.nexsroot.applylock:
|
|
1715
1719
|
|
|
1720
|
+
logger.debug(f'HANDOFF: Obtained nexus applylock ahaname={ahaname}')
|
|
1716
1721
|
indx = await self.getNexsIndx()
|
|
1722
|
+
|
|
1723
|
+
logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}')
|
|
1717
1724
|
if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover
|
|
1718
1725
|
mndx = await cell.getNexsIndx()
|
|
1719
1726
|
mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.'
|
|
1720
1727
|
raise s_exc.NotReady(mesg=mesg)
|
|
1721
1728
|
|
|
1729
|
+
logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion ahaname={ahaname}')
|
|
1722
1730
|
await cell.promote()
|
|
1731
|
+
|
|
1732
|
+
logger.debug(f'HANDOFF: Setting the service as inactive ahaname={ahaname}')
|
|
1723
1733
|
await self.setCellActive(False)
|
|
1724
1734
|
|
|
1735
|
+
logger.debug(f'HANDOFF: Configuring service to use the new leader as its mirror ahaname={ahaname}')
|
|
1725
1736
|
self.modCellConf({'mirror': turl})
|
|
1737
|
+
|
|
1738
|
+
logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}')
|
|
1726
1739
|
await self.nexsroot.startup()
|
|
1727
1740
|
|
|
1741
|
+
logger.debug(f'HANDOFF: Released nexus applylock ahaname={ahaname}')
|
|
1742
|
+
logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}')
|
|
1743
|
+
|
|
1728
1744
|
async def reqAhaProxy(self, timeout=None):
|
|
1729
1745
|
if self.ahaclient is None:
|
|
1730
1746
|
mesg = 'AHA is not configured on this service.'
|
|
@@ -1855,18 +1871,32 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
1855
1871
|
return self.isactive
|
|
1856
1872
|
|
|
1857
1873
|
async def setCellActive(self, active):
|
|
1874
|
+
|
|
1875
|
+
if active == self.isactive:
|
|
1876
|
+
return
|
|
1877
|
+
|
|
1858
1878
|
self.isactive = active
|
|
1859
1879
|
|
|
1860
1880
|
if self.isactive:
|
|
1881
|
+
self.activebase = await s_base.Base.anit()
|
|
1882
|
+
self.onfini(self.activebase)
|
|
1861
1883
|
self._fireActiveCoros()
|
|
1862
1884
|
await self._execCellUpdates()
|
|
1863
1885
|
await self.initServiceActive()
|
|
1864
1886
|
else:
|
|
1865
1887
|
await self._killActiveCoros()
|
|
1888
|
+
await self.activebase.fini()
|
|
1889
|
+
self.activebase = None
|
|
1866
1890
|
await self.initServicePassive()
|
|
1867
1891
|
|
|
1868
1892
|
await self._setAhaActive()
|
|
1869
1893
|
|
|
1894
|
+
def runActiveTask(self, coro):
|
|
1895
|
+
# an API for active coroutines to use when running an
|
|
1896
|
+
# ephemeral task which should be automatically torn down
|
|
1897
|
+
# if the cell becomes inactive
|
|
1898
|
+
return self.activebase.schedCoro(coro)
|
|
1899
|
+
|
|
1870
1900
|
async def initServiceActive(self): # pragma: no cover
|
|
1871
1901
|
pass
|
|
1872
1902
|
|
|
@@ -3654,7 +3684,8 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
|
|
|
3654
3684
|
|
|
3655
3685
|
await self.ahaclient.waitready()
|
|
3656
3686
|
|
|
3657
|
-
|
|
3687
|
+
proxy = await self.ahaclient.proxy(timeout=5)
|
|
3688
|
+
mirrors = await proxy.getAhaSvcMirrors(self.ahasvcname)
|
|
3658
3689
|
if mirrors is None:
|
|
3659
3690
|
mesg = 'Service must be configured with AHA to enumerate mirror URLs'
|
|
3660
3691
|
raise s_exc.NoSuchName(mesg=mesg, name=self.ahasvcname)
|
synapse/lib/certdir.py
CHANGED
|
@@ -1501,6 +1501,11 @@ class CertDir:
|
|
|
1501
1501
|
return c_rsa.generate_private_key(65537, self.crypto_numbits)
|
|
1502
1502
|
|
|
1503
1503
|
def _genCertBuilder(self, name: str, pubkey: c_types.PublicKeyTypes) -> c_x509.CertificateBuilder:
|
|
1504
|
+
|
|
1505
|
+
if not 1 <= len(name) <= 64:
|
|
1506
|
+
mesg = f'Certificate name values must be between 1-64 characters. got name={name}, len={len(name)}'
|
|
1507
|
+
raise s_exc.CryptoErr(mesg=mesg)
|
|
1508
|
+
|
|
1504
1509
|
builder = c_x509.CertificateBuilder()
|
|
1505
1510
|
builder = builder.subject_name(c_x509.Name([
|
|
1506
1511
|
c_x509.NameAttribute(c_x509.NameOID.COMMON_NAME, name),
|
|
@@ -1515,6 +1520,10 @@ class CertDir:
|
|
|
1515
1520
|
|
|
1516
1521
|
def _genPkeyCsr(self, name: str, mode: str, outp: OutPutOrNone = None) -> bytes:
|
|
1517
1522
|
|
|
1523
|
+
if not 1 <= len(name) <= 64:
|
|
1524
|
+
mesg = f'CSR name values must be between 1-64 characters. got name={name}, len={len(name)}'
|
|
1525
|
+
raise s_exc.CryptoErr(mesg=mesg)
|
|
1526
|
+
|
|
1518
1527
|
pkey = self._genPrivKey()
|
|
1519
1528
|
|
|
1520
1529
|
builder = c_x509.CertificateSigningRequestBuilder()
|
synapse/lib/hiveauth.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
|
+
import dataclasses
|
|
3
|
+
|
|
4
|
+
from typing import Union
|
|
2
5
|
|
|
3
6
|
import synapse.exc as s_exc
|
|
4
7
|
import synapse.common as s_common
|
|
@@ -38,6 +41,45 @@ def textFromRule(rule):
|
|
|
38
41
|
text = '!' + text
|
|
39
42
|
return text
|
|
40
43
|
|
|
44
|
+
@dataclasses.dataclass(slots=True)
|
|
45
|
+
class _allowedReason:
|
|
46
|
+
value: Union[bool | None]
|
|
47
|
+
default: bool = False
|
|
48
|
+
isadmin: bool = False
|
|
49
|
+
islocked: bool = False
|
|
50
|
+
gateiden: Union[str | None] = None
|
|
51
|
+
roleiden: Union[str | None] = None
|
|
52
|
+
rolename: Union[str | None] = None
|
|
53
|
+
rule: tuple = ()
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def mesg(self):
|
|
57
|
+
if self.islocked:
|
|
58
|
+
return 'The user is locked.'
|
|
59
|
+
if self.default:
|
|
60
|
+
return 'No matching rule found.'
|
|
61
|
+
|
|
62
|
+
if self.isadmin:
|
|
63
|
+
if self.gateiden:
|
|
64
|
+
return f'The user is an admin of auth gate {self.gateiden}.'
|
|
65
|
+
return 'The user is a global admin.'
|
|
66
|
+
|
|
67
|
+
if self.rule:
|
|
68
|
+
rt = textFromRule((self.value, self.rule))
|
|
69
|
+
if self.gateiden:
|
|
70
|
+
if self.roleiden:
|
|
71
|
+
m = f'Matched role rule ({rt}) for role {self.rolename} on gate {self.gateiden}.'
|
|
72
|
+
else:
|
|
73
|
+
m = f'Matched user rule ({rt}) on gate {self.gateiden}.'
|
|
74
|
+
else:
|
|
75
|
+
if self.roleiden:
|
|
76
|
+
m = f'Matched role rule ({rt}) for role {self.rolename}.'
|
|
77
|
+
else:
|
|
78
|
+
m = f'Matched user rule ({rt}).'
|
|
79
|
+
return m
|
|
80
|
+
|
|
81
|
+
return 'No matching rule found.'
|
|
82
|
+
|
|
41
83
|
class Auth(s_nexus.Pusher):
|
|
42
84
|
'''
|
|
43
85
|
Auth is a user authentication and authorization stored in a Hive. Users
|
|
@@ -827,7 +869,7 @@ class HiveRole(HiveRuler):
|
|
|
827
869
|
return allow
|
|
828
870
|
return default
|
|
829
871
|
|
|
830
|
-
# 2. check
|
|
872
|
+
# 2. check role rules
|
|
831
873
|
for allow, path in self.info.get('rules', ()):
|
|
832
874
|
if perm[:len(path)] == path:
|
|
833
875
|
return allow
|
|
@@ -858,6 +900,7 @@ class HiveUser(HiveRuler):
|
|
|
858
900
|
self.vars = await varz.dict(nexs=True)
|
|
859
901
|
|
|
860
902
|
self.permcache = s_cache.FixedCache(self._allowed)
|
|
903
|
+
self.allowedcache = s_cache.FixedCache(self._getAllowedReason)
|
|
861
904
|
|
|
862
905
|
def pack(self, packroles=False):
|
|
863
906
|
|
|
@@ -903,6 +946,9 @@ class HiveUser(HiveRuler):
|
|
|
903
946
|
return self.permcache.get((perm, default, gateiden))
|
|
904
947
|
|
|
905
948
|
def _allowed(self, pkey):
|
|
949
|
+
'''
|
|
950
|
+
NOTE: This must remain in sync with any changes to _getAllowedReason()!
|
|
951
|
+
'''
|
|
906
952
|
|
|
907
953
|
perm, default, gateiden = pkey
|
|
908
954
|
|
|
@@ -951,18 +997,23 @@ class HiveUser(HiveRuler):
|
|
|
951
997
|
|
|
952
998
|
return default
|
|
953
999
|
|
|
954
|
-
def getAllowedReason(self, perm,
|
|
1000
|
+
def getAllowedReason(self, perm, default=None, gateiden=None):
|
|
1001
|
+
'''
|
|
1002
|
+
A routine which will return a tuple of (allowed, info).
|
|
955
1003
|
'''
|
|
956
|
-
|
|
957
|
-
|
|
1004
|
+
perm = tuple(perm)
|
|
1005
|
+
return self.allowedcache.get((perm, default, gateiden))
|
|
958
1006
|
|
|
1007
|
+
def _getAllowedReason(self, pkey):
|
|
1008
|
+
'''
|
|
959
1009
|
NOTE: This must remain in sync with any changes to _allowed()!
|
|
960
1010
|
'''
|
|
1011
|
+
perm, default, gateiden = pkey
|
|
961
1012
|
if self.info.get('locked'):
|
|
962
|
-
return (False,
|
|
1013
|
+
return _allowedReason(False, islocked=True)
|
|
963
1014
|
|
|
964
1015
|
if self.info.get('admin'):
|
|
965
|
-
return (True,
|
|
1016
|
+
return _allowedReason(True, isadmin=True)
|
|
966
1017
|
|
|
967
1018
|
# 1. check authgate user rules
|
|
968
1019
|
if gateiden is not None:
|
|
@@ -971,16 +1022,16 @@ class HiveUser(HiveRuler):
|
|
|
971
1022
|
if info is not None:
|
|
972
1023
|
|
|
973
1024
|
if info.get('admin'):
|
|
974
|
-
return (True,
|
|
1025
|
+
return _allowedReason(True, isadmin=True, gateiden=gateiden)
|
|
975
1026
|
|
|
976
1027
|
for allow, path in info.get('rules', ()):
|
|
977
1028
|
if perm[:len(path)] == path:
|
|
978
|
-
return (allow,
|
|
1029
|
+
return _allowedReason(allow, gateiden=gateiden, rule=path)
|
|
979
1030
|
|
|
980
1031
|
# 2. check user rules
|
|
981
1032
|
for allow, path in self.info.get('rules', ()):
|
|
982
1033
|
if perm[:len(path)] == path:
|
|
983
|
-
return (allow,
|
|
1034
|
+
return _allowedReason(allow, rule=path)
|
|
984
1035
|
|
|
985
1036
|
# 3. check authgate role rules
|
|
986
1037
|
if gateiden is not None:
|
|
@@ -993,18 +1044,20 @@ class HiveUser(HiveRuler):
|
|
|
993
1044
|
|
|
994
1045
|
for allow, path in info.get('rules', ()):
|
|
995
1046
|
if perm[:len(path)] == path:
|
|
996
|
-
return (allow,
|
|
1047
|
+
return _allowedReason(allow, gateiden=gateiden, roleiden=role.iden, rolename=role.name,
|
|
1048
|
+
rule=path)
|
|
997
1049
|
|
|
998
1050
|
# 4. check role rules
|
|
999
1051
|
for role in self.getRoles():
|
|
1000
1052
|
for allow, path in role.info.get('rules', ()):
|
|
1001
1053
|
if perm[:len(path)] == path:
|
|
1002
|
-
return (allow,
|
|
1054
|
+
return _allowedReason(allow, roleiden=role.iden, rolename=role.name, rule=path)
|
|
1003
1055
|
|
|
1004
|
-
return (default,
|
|
1056
|
+
return _allowedReason(default, default=True)
|
|
1005
1057
|
|
|
1006
1058
|
def clearAuthCache(self):
|
|
1007
1059
|
self.permcache.clear()
|
|
1060
|
+
self.allowedcache.clear()
|
|
1008
1061
|
|
|
1009
1062
|
async def genGateInfo(self, gateiden):
|
|
1010
1063
|
info = self.authgates.get(gateiden)
|
synapse/lib/httpapi.py
CHANGED
synapse/lib/modelrev.py
CHANGED
|
@@ -8,7 +8,7 @@ import synapse.lib.layer as s_layer
|
|
|
8
8
|
|
|
9
9
|
logger = logging.getLogger(__name__)
|
|
10
10
|
|
|
11
|
-
maxvers = (0, 2,
|
|
11
|
+
maxvers = (0, 2, 24)
|
|
12
12
|
|
|
13
13
|
class ModelRev:
|
|
14
14
|
|
|
@@ -37,6 +37,7 @@ class ModelRev:
|
|
|
37
37
|
((0, 2, 21), self.revModel_0_2_21),
|
|
38
38
|
((0, 2, 22), self.revModel_0_2_22),
|
|
39
39
|
((0, 2, 23), self.revModel_0_2_23),
|
|
40
|
+
((0, 2, 24), self.revModel_0_2_24),
|
|
40
41
|
)
|
|
41
42
|
|
|
42
43
|
async def _uniqSortArray(self, todoprops, layers):
|
|
@@ -733,6 +734,29 @@ class ModelRev:
|
|
|
733
734
|
async def revModel_0_2_23(self, layers):
|
|
734
735
|
await self._normFormSubs(layers, 'inet:ipv6')
|
|
735
736
|
|
|
737
|
+
async def revModel_0_2_24(self, layers):
|
|
738
|
+
await self._normPropValu(layers, 'risk:mitigation:name')
|
|
739
|
+
await self._normPropValu(layers, 'it:mitre:attack:technique:name')
|
|
740
|
+
await self._normPropValu(layers, 'it:mitre:attack:mitigation:name')
|
|
741
|
+
|
|
742
|
+
formprops = {}
|
|
743
|
+
for prop in self.core.model.getPropsByType('velocity'):
|
|
744
|
+
formname = prop.form.name
|
|
745
|
+
if formname not in formprops:
|
|
746
|
+
formprops[formname] = []
|
|
747
|
+
|
|
748
|
+
formprops[formname].append(prop)
|
|
749
|
+
|
|
750
|
+
for prop in self.core.model.getArrayPropsByType('velocity'):
|
|
751
|
+
formname = prop.form.name
|
|
752
|
+
if formname not in formprops:
|
|
753
|
+
formprops[formname] = []
|
|
754
|
+
|
|
755
|
+
formprops[formname].append(prop)
|
|
756
|
+
|
|
757
|
+
for form, props in formprops.items():
|
|
758
|
+
await self._normVelocityProps(layers, form, props)
|
|
759
|
+
|
|
736
760
|
async def runStorm(self, text, opts=None):
|
|
737
761
|
'''
|
|
738
762
|
Run storm code in a schedcoro and log the output messages.
|
|
@@ -759,52 +783,50 @@ class ModelRev:
|
|
|
759
783
|
|
|
760
784
|
async def revCoreLayers(self):
|
|
761
785
|
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
version = self.revs[-1][0] if self.revs else maxvers
|
|
786
|
+
version = self.revs[-1][0] if self.revs else maxvers
|
|
765
787
|
|
|
766
|
-
|
|
767
|
-
|
|
788
|
+
# do a first pass to detect layers at the wrong version
|
|
789
|
+
# that we are not able to rev ourselves and bail...
|
|
768
790
|
|
|
769
|
-
|
|
770
|
-
|
|
791
|
+
layers = []
|
|
792
|
+
for layr in self.core.layers.values():
|
|
771
793
|
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
794
|
+
if layr.fresh:
|
|
795
|
+
await layr.setModelVers(version)
|
|
796
|
+
continue
|
|
775
797
|
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
798
|
+
vers = await layr.getModelVers()
|
|
799
|
+
if vers == version:
|
|
800
|
+
continue
|
|
779
801
|
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
802
|
+
if not layr.canrev and vers != version:
|
|
803
|
+
mesg = f'layer {layr.__class__.__name__} {layr.iden} ({layr.dirn}) can not be updated.'
|
|
804
|
+
raise s_exc.CantRevLayer(layer=layr.iden, mesg=mesg, curv=version, layv=vers)
|
|
783
805
|
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
806
|
+
if vers > version:
|
|
807
|
+
mesg = f'layer {layr.__class__.__name__} {layr.iden} ({layr.dirn}) is from the future!'
|
|
808
|
+
raise s_exc.CantRevLayer(layer=layr.iden, mesg=mesg, curv=version, layv=vers)
|
|
787
809
|
|
|
788
|
-
|
|
789
|
-
|
|
810
|
+
# realistically all layers are probably at the same version... but...
|
|
811
|
+
layers.append(layr)
|
|
790
812
|
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
813
|
+
# got anything to do?
|
|
814
|
+
if not layers:
|
|
815
|
+
return
|
|
794
816
|
|
|
795
|
-
|
|
817
|
+
for revvers, revmeth in self.revs:
|
|
796
818
|
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
819
|
+
todo = [lyr for lyr in layers if not lyr.ismirror and await lyr.getModelVers() < revvers]
|
|
820
|
+
if not todo:
|
|
821
|
+
continue
|
|
800
822
|
|
|
801
|
-
|
|
823
|
+
logger.warning(f'beginning model migration -> {revvers}')
|
|
802
824
|
|
|
803
|
-
|
|
825
|
+
await revmeth(todo)
|
|
804
826
|
|
|
805
|
-
|
|
827
|
+
[await lyr.setModelVers(revvers) for lyr in todo]
|
|
806
828
|
|
|
807
|
-
|
|
829
|
+
logger.warning('...model migrations complete!')
|
|
808
830
|
|
|
809
831
|
async def _normPropValu(self, layers, propfull):
|
|
810
832
|
|
|
@@ -850,6 +872,72 @@ class ModelRev:
|
|
|
850
872
|
if nodeedits:
|
|
851
873
|
await save()
|
|
852
874
|
|
|
875
|
+
async def _normVelocityProps(self, layers, form, props):
|
|
876
|
+
|
|
877
|
+
meta = {'time': s_common.now(), 'user': self.core.auth.rootuser.iden}
|
|
878
|
+
|
|
879
|
+
nodeedits = []
|
|
880
|
+
for layr in layers:
|
|
881
|
+
|
|
882
|
+
async def save():
|
|
883
|
+
await layr.storNodeEdits(nodeedits, meta)
|
|
884
|
+
nodeedits.clear()
|
|
885
|
+
|
|
886
|
+
async for buid, formvalu in layr.iterFormRows(form):
|
|
887
|
+
sode = layr._getStorNode(buid)
|
|
888
|
+
if (nodeprops := sode.get('props')) is None:
|
|
889
|
+
continue
|
|
890
|
+
|
|
891
|
+
for prop in props:
|
|
892
|
+
if (curv := nodeprops.get(prop.name)) is None:
|
|
893
|
+
continue
|
|
894
|
+
|
|
895
|
+
propvalu = curv[0]
|
|
896
|
+
if prop.type.isarray:
|
|
897
|
+
hasfloat = False
|
|
898
|
+
strvalu = []
|
|
899
|
+
for valu in propvalu:
|
|
900
|
+
if isinstance(valu, float):
|
|
901
|
+
strvalu.append(str(valu))
|
|
902
|
+
hasfloat = True
|
|
903
|
+
|
|
904
|
+
if not hasfloat:
|
|
905
|
+
continue
|
|
906
|
+
else:
|
|
907
|
+
if not isinstance(propvalu, float):
|
|
908
|
+
continue
|
|
909
|
+
strvalu = str(propvalu)
|
|
910
|
+
|
|
911
|
+
nodeprops.pop(prop.name)
|
|
912
|
+
|
|
913
|
+
try:
|
|
914
|
+
norm, info = prop.type.norm(strvalu)
|
|
915
|
+
except s_exc.BadTypeValu as e:
|
|
916
|
+
nodeedits.append(
|
|
917
|
+
(buid, form, (
|
|
918
|
+
(s_layer.EDIT_NODEDATA_SET, (f'_migrated:{prop.full}', propvalu, None), ()),
|
|
919
|
+
(s_layer.EDIT_PROP_DEL, (prop.name, propvalu, prop.type.stortype), ()),
|
|
920
|
+
)),
|
|
921
|
+
)
|
|
922
|
+
|
|
923
|
+
oldm = e.errinfo.get('mesg')
|
|
924
|
+
iden = s_common.ehex(buid)
|
|
925
|
+
logger.warning(f'error re-norming {prop.full}={propvalu} (layer: {layr.iden}, node: {iden}): {oldm}',
|
|
926
|
+
extra={'synapse': {'node': iden, 'layer': layr.iden}})
|
|
927
|
+
continue
|
|
928
|
+
|
|
929
|
+
nodeedits.append(
|
|
930
|
+
(buid, form, (
|
|
931
|
+
(s_layer.EDIT_PROP_SET, (prop.name, norm, propvalu, prop.type.stortype), ()),
|
|
932
|
+
)),
|
|
933
|
+
)
|
|
934
|
+
|
|
935
|
+
if len(nodeedits) >= 1000: # pragma: no cover
|
|
936
|
+
await save()
|
|
937
|
+
|
|
938
|
+
if nodeedits:
|
|
939
|
+
await save()
|
|
940
|
+
|
|
853
941
|
async def _updatePropStortype(self, layers, propfull):
|
|
854
942
|
|
|
855
943
|
meta = {'time': s_common.now(), 'user': self.core.auth.rootuser.iden}
|