synapse 2.171.0__py311-none-any.whl → 2.173.1__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/common.py +20 -0
- synapse/cortex.py +86 -4
- synapse/lib/agenda.py +13 -7
- synapse/lib/ast.py +9 -8
- synapse/lib/cache.py +2 -2
- synapse/lib/cell.py +5 -0
- synapse/lib/coro.py +12 -0
- synapse/lib/hiveauth.py +81 -4
- synapse/lib/layer.py +124 -84
- synapse/lib/lmdbslab.py +17 -10
- synapse/lib/node.py +1 -1
- synapse/lib/slabseqn.py +11 -5
- synapse/lib/storm.py +7 -71
- synapse/lib/stormhttp.py +1 -1
- synapse/lib/stormlib/auth.py +19 -0
- synapse/lib/stormlib/cell.py +42 -4
- synapse/lib/stormlib/compression.py +6 -6
- synapse/lib/stormlib/env.py +50 -0
- synapse/lib/stormlib/gen.py +1 -1
- synapse/lib/stormlib/model.py +1 -1
- synapse/lib/stormtypes.py +35 -11
- synapse/lib/types.py +6 -6
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +6 -12
- synapse/models/base.py +13 -0
- synapse/models/biz.py +14 -0
- synapse/models/economic.py +3 -0
- synapse/models/inet.py +474 -4
- synapse/models/infotech.py +163 -22
- synapse/models/orgs.py +22 -2
- synapse/models/person.py +5 -2
- synapse/models/planning.py +5 -0
- synapse/models/risk.py +15 -1
- synapse/models/transport.py +1 -1
- synapse/tests/test_common.py +15 -0
- synapse/tests/test_lib_ast.py +2 -1
- synapse/tests/test_lib_hiveauth.py +139 -1
- synapse/tests/test_lib_layer.py +207 -44
- synapse/tests/test_lib_lmdbslab.py +13 -0
- synapse/tests/test_lib_stormlib_auth.py +22 -0
- synapse/tests/test_lib_stormlib_cell.py +47 -0
- synapse/tests/test_lib_stormlib_env.py +25 -0
- synapse/tests/test_lib_view.py +9 -9
- synapse/tests/test_model_base.py +5 -3
- synapse/tests/test_model_economic.py +4 -0
- synapse/tests/test_model_inet.py +405 -1
- synapse/tests/test_model_infotech.py +135 -3
- synapse/tests/test_model_orgs.py +14 -2
- synapse/tests/test_model_person.py +2 -0
- synapse/tests/test_model_risk.py +8 -0
- synapse/tests/test_tools_storm.py +46 -8
- synapse/tools/storm.py +14 -6
- {synapse-2.171.0.dist-info → synapse-2.173.1.dist-info}/METADATA +1 -1
- {synapse-2.171.0.dist-info → synapse-2.173.1.dist-info}/RECORD +57 -55
- {synapse-2.171.0.dist-info → synapse-2.173.1.dist-info}/WHEEL +1 -1
- {synapse-2.171.0.dist-info → synapse-2.173.1.dist-info}/LICENSE +0 -0
- {synapse-2.171.0.dist-info → synapse-2.173.1.dist-info}/top_level.txt +0 -0
synapse/lib/layer.py
CHANGED
|
@@ -75,6 +75,7 @@ import synapse.telepath as s_telepath
|
|
|
75
75
|
|
|
76
76
|
import synapse.lib.gis as s_gis
|
|
77
77
|
import synapse.lib.cell as s_cell
|
|
78
|
+
import synapse.lib.coro as s_coro
|
|
78
79
|
import synapse.lib.cache as s_cache
|
|
79
80
|
import synapse.lib.nexus as s_nexus
|
|
80
81
|
import synapse.lib.queue as s_queue
|
|
@@ -157,14 +158,14 @@ class LayerApi(s_cell.CellApi):
|
|
|
157
158
|
|
|
158
159
|
await self.layr.storNodeEditsNoLift(nodeedits, meta)
|
|
159
160
|
|
|
160
|
-
async def syncNodeEdits(self, offs, wait=True):
|
|
161
|
+
async def syncNodeEdits(self, offs, wait=True, reverse=False):
|
|
161
162
|
'''
|
|
162
163
|
Yield (offs, nodeedits) tuples from the nodeedit log starting from the given offset.
|
|
163
164
|
|
|
164
165
|
Once caught up with storage, yield them in realtime.
|
|
165
166
|
'''
|
|
166
167
|
await self._reqUserAllowed(self.liftperm)
|
|
167
|
-
async for item in self.layr.syncNodeEdits(offs, wait=wait):
|
|
168
|
+
async for item in self.layr.syncNodeEdits(offs, wait=wait, reverse=reverse):
|
|
168
169
|
yield item
|
|
169
170
|
await asyncio.sleep(0)
|
|
170
171
|
|
|
@@ -3816,7 +3817,7 @@ class Layer(s_nexus.Pusher):
|
|
|
3816
3817
|
|
|
3817
3818
|
async def getEdgeVerbs(self):
|
|
3818
3819
|
|
|
3819
|
-
for lkey in self.layrslab.scanKeys(db=self.byverb):
|
|
3820
|
+
for lkey in self.layrslab.scanKeys(db=self.byverb, nodup=True):
|
|
3820
3821
|
yield lkey.decode()
|
|
3821
3822
|
|
|
3822
3823
|
async def getEdges(self, verb=None):
|
|
@@ -4088,11 +4089,126 @@ class Layer(s_nexus.Pusher):
|
|
|
4088
4089
|
'''
|
|
4089
4090
|
Return a generator of all a buid's node data keys
|
|
4090
4091
|
'''
|
|
4091
|
-
for lkey in self.dataslab.scanKeysByPref(buid, db=self.nodedata):
|
|
4092
|
+
for lkey in self.dataslab.scanKeysByPref(buid, db=self.nodedata, nodup=True):
|
|
4092
4093
|
abrv = lkey[32:]
|
|
4093
4094
|
prop = self.getAbrvProp(abrv)
|
|
4094
4095
|
yield prop[0]
|
|
4095
4096
|
|
|
4097
|
+
async def confirmLayerEditPerms(self, user, gateiden, delete=False):
|
|
4098
|
+
if delete:
|
|
4099
|
+
perm_forms = ('node', 'del')
|
|
4100
|
+
perm_props = ('node', 'prop', 'del')
|
|
4101
|
+
perm_tags = ('node', 'tag', 'del')
|
|
4102
|
+
perm_ndata = ('node', 'data', 'pop')
|
|
4103
|
+
perm_edges = ('node', 'edge', 'del')
|
|
4104
|
+
else:
|
|
4105
|
+
perm_forms = ('node', 'add')
|
|
4106
|
+
perm_props = ('node', 'prop', 'set')
|
|
4107
|
+
perm_tags = ('node', 'tag', 'add')
|
|
4108
|
+
perm_ndata = ('node', 'data', 'set')
|
|
4109
|
+
perm_edges = ('node', 'edge', 'add')
|
|
4110
|
+
|
|
4111
|
+
if user.allowed(('node',), gateiden=gateiden, deepdeny=True):
|
|
4112
|
+
return
|
|
4113
|
+
|
|
4114
|
+
allow_forms = user.allowed(perm_forms, gateiden=gateiden, deepdeny=True)
|
|
4115
|
+
allow_props = user.allowed(perm_props, gateiden=gateiden, deepdeny=True)
|
|
4116
|
+
allow_tags = user.allowed(perm_tags, gateiden=gateiden, deepdeny=True)
|
|
4117
|
+
allow_ndata = user.allowed(perm_ndata, gateiden=gateiden, deepdeny=True)
|
|
4118
|
+
allow_edges = user.allowed(perm_edges, gateiden=gateiden, deepdeny=True)
|
|
4119
|
+
|
|
4120
|
+
if all((allow_forms, allow_props, allow_tags, allow_ndata, allow_edges)):
|
|
4121
|
+
return
|
|
4122
|
+
|
|
4123
|
+
# nodes & props
|
|
4124
|
+
if not allow_forms or not allow_props:
|
|
4125
|
+
async for byts, abrv in s_coro.pause(self.propabrv.slab.scanByFull(db=self.propabrv.name2abrv)):
|
|
4126
|
+
form, prop = s_msgpack.un(byts)
|
|
4127
|
+
if form is None: # pragma: no cover
|
|
4128
|
+
continue
|
|
4129
|
+
|
|
4130
|
+
if self.layrslab.prefexists(abrv, db=self.byprop):
|
|
4131
|
+
if prop and not allow_props:
|
|
4132
|
+
realform = self.core.model.form(form)
|
|
4133
|
+
if not realform: # pragma: no cover
|
|
4134
|
+
mesg = f'Invalid form: {form}'
|
|
4135
|
+
raise s_exc.NoSuchForm(mesg=mesg, form=form)
|
|
4136
|
+
|
|
4137
|
+
realprop = realform.prop(prop)
|
|
4138
|
+
if not realprop: # pragma: no cover
|
|
4139
|
+
mesg = f'Invalid prop: {form}:{prop}'
|
|
4140
|
+
raise s_exc.NoSuchProp(mesg=mesg, form=form, prop=prop)
|
|
4141
|
+
|
|
4142
|
+
if delete:
|
|
4143
|
+
self.core.confirmPropDel(user, realprop, gateiden)
|
|
4144
|
+
else:
|
|
4145
|
+
self.core.confirmPropSet(user, realprop, gateiden)
|
|
4146
|
+
|
|
4147
|
+
elif not prop and not allow_forms:
|
|
4148
|
+
user.confirm(perm_forms + (form,), gateiden=gateiden)
|
|
4149
|
+
|
|
4150
|
+
# tagprops
|
|
4151
|
+
if not allow_tags:
|
|
4152
|
+
async for byts, abrv in s_coro.pause(self.tagpropabrv.slab.scanByFull(db=self.tagpropabrv.name2abrv)):
|
|
4153
|
+
info = s_msgpack.un(byts)
|
|
4154
|
+
if None in info or len(info) != 3:
|
|
4155
|
+
continue
|
|
4156
|
+
|
|
4157
|
+
if self.layrslab.prefexists(abrv, db=self.bytagprop):
|
|
4158
|
+
perm = perm_tags + tuple(info[1].split('.'))
|
|
4159
|
+
user.confirm(perm, gateiden=gateiden)
|
|
4160
|
+
|
|
4161
|
+
# nodedata
|
|
4162
|
+
if not allow_ndata:
|
|
4163
|
+
async for abrv in s_coro.pause(self.dataslab.scanKeys(db=self.dataname, nodup=True)):
|
|
4164
|
+
name, _ = self.getAbrvProp(abrv)
|
|
4165
|
+
perm = perm_ndata + (name,)
|
|
4166
|
+
user.confirm(perm, gateiden=gateiden)
|
|
4167
|
+
|
|
4168
|
+
# edges
|
|
4169
|
+
if not allow_edges:
|
|
4170
|
+
async for verb in s_coro.pause(self.layrslab.scanKeys(db=self.byverb, nodup=True)):
|
|
4171
|
+
perm = perm_edges + (verb.decode(),)
|
|
4172
|
+
user.confirm(perm, gateiden=gateiden)
|
|
4173
|
+
|
|
4174
|
+
# tags
|
|
4175
|
+
# NB: tag perms should be yielded for every leaf on every node in the layer
|
|
4176
|
+
if not allow_tags:
|
|
4177
|
+
async with self.core.getSpooledDict() as tags:
|
|
4178
|
+
|
|
4179
|
+
# Collect all tag abrvs for all nodes in the layer
|
|
4180
|
+
async for lkey, buid in s_coro.pause(self.layrslab.scanByFull(db=self.bytag)):
|
|
4181
|
+
abrv = lkey[:8]
|
|
4182
|
+
abrvs = list(tags.get(buid, []))
|
|
4183
|
+
abrvs.append(abrv)
|
|
4184
|
+
await tags.set(buid, abrvs)
|
|
4185
|
+
|
|
4186
|
+
# Iterate over each node and it's tags
|
|
4187
|
+
async for buid, abrvs in s_coro.pause(tags.items()):
|
|
4188
|
+
seen = {}
|
|
4189
|
+
|
|
4190
|
+
if len(abrvs) == 1:
|
|
4191
|
+
# Easy optimization: If there's only one tag abrv, then it's a
|
|
4192
|
+
# leaf by default
|
|
4193
|
+
name = self.tagabrv.abrvToName(abrv)
|
|
4194
|
+
key = tuple(name.split('.'))
|
|
4195
|
+
perm = perm_tags + key
|
|
4196
|
+
user.confirm(perm, gateiden=gateiden)
|
|
4197
|
+
|
|
4198
|
+
else:
|
|
4199
|
+
for abrv in abrvs:
|
|
4200
|
+
name = self.tagabrv.abrvToName(abrv)
|
|
4201
|
+
parts = tuple(name.split('.'))
|
|
4202
|
+
for idx in range(1, len(parts) + 1):
|
|
4203
|
+
key = tuple(parts[:idx])
|
|
4204
|
+
seen.setdefault(key, 0)
|
|
4205
|
+
seen[key] += 1
|
|
4206
|
+
|
|
4207
|
+
for key, count in seen.items():
|
|
4208
|
+
if count == 1:
|
|
4209
|
+
perm = perm_tags + key
|
|
4210
|
+
user.confirm(perm, gateiden=gateiden)
|
|
4211
|
+
|
|
4096
4212
|
async def iterLayerNodeEdits(self):
|
|
4097
4213
|
'''
|
|
4098
4214
|
Scan the full layer and yield artificial sets of nodeedits.
|
|
@@ -4291,7 +4407,7 @@ class Layer(s_nexus.Pusher):
|
|
|
4291
4407
|
for offs, (edits, meta) in self.nodeeditlog.iterBack(offs):
|
|
4292
4408
|
yield (offs, edits, meta)
|
|
4293
4409
|
|
|
4294
|
-
async def syncNodeEdits2(self, offs, wait=True):
|
|
4410
|
+
async def syncNodeEdits2(self, offs, wait=True, reverse=False):
|
|
4295
4411
|
'''
|
|
4296
4412
|
Once caught up with storage, yield them in realtime.
|
|
4297
4413
|
|
|
@@ -4301,7 +4417,7 @@ class Layer(s_nexus.Pusher):
|
|
|
4301
4417
|
if not self.logedits:
|
|
4302
4418
|
return
|
|
4303
4419
|
|
|
4304
|
-
for offi, (nodeedits, meta) in self.nodeeditlog.iter(offs):
|
|
4420
|
+
for offi, (nodeedits, meta) in self.nodeeditlog.iter(offs, reverse=reverse):
|
|
4305
4421
|
yield (offi, nodeedits, meta)
|
|
4306
4422
|
|
|
4307
4423
|
if wait:
|
|
@@ -4309,11 +4425,11 @@ class Layer(s_nexus.Pusher):
|
|
|
4309
4425
|
async for item in wind:
|
|
4310
4426
|
yield item
|
|
4311
4427
|
|
|
4312
|
-
async def syncNodeEdits(self, offs, wait=True):
|
|
4428
|
+
async def syncNodeEdits(self, offs, wait=True, reverse=False):
|
|
4313
4429
|
'''
|
|
4314
4430
|
Identical to syncNodeEdits2, but doesn't yield meta
|
|
4315
4431
|
'''
|
|
4316
|
-
async for offi, nodeedits, _meta in self.syncNodeEdits2(offs, wait=wait):
|
|
4432
|
+
async for offi, nodeedits, _meta in self.syncNodeEdits2(offs, wait=wait, reverse=reverse):
|
|
4317
4433
|
yield (offi, nodeedits)
|
|
4318
4434
|
|
|
4319
4435
|
async def syncIndexEvents(self, offs, matchdef, wait=True):
|
|
@@ -4450,79 +4566,3 @@ def getFlatEdits(nodeedits):
|
|
|
4450
4566
|
addedits(buid, form, edits)
|
|
4451
4567
|
|
|
4452
4568
|
return [(k[0], k[1], v) for (k, v) in editsbynode.items()]
|
|
4453
|
-
|
|
4454
|
-
def getNodeEditPerms(nodeedits):
|
|
4455
|
-
'''
|
|
4456
|
-
Yields (offs, perm) tuples that can be used in user.allowed()
|
|
4457
|
-
'''
|
|
4458
|
-
tags = []
|
|
4459
|
-
tagadds = []
|
|
4460
|
-
|
|
4461
|
-
for nodeoffs, (buid, form, edits) in enumerate(nodeedits):
|
|
4462
|
-
|
|
4463
|
-
tags.clear()
|
|
4464
|
-
tagadds.clear()
|
|
4465
|
-
|
|
4466
|
-
for editoffs, (edit, info, _) in enumerate(edits):
|
|
4467
|
-
|
|
4468
|
-
permoffs = (nodeoffs, editoffs)
|
|
4469
|
-
|
|
4470
|
-
if edit == EDIT_NODE_ADD:
|
|
4471
|
-
yield (permoffs, ('node', 'add', form))
|
|
4472
|
-
continue
|
|
4473
|
-
|
|
4474
|
-
if edit == EDIT_NODE_DEL:
|
|
4475
|
-
yield (permoffs, ('node', 'del', form))
|
|
4476
|
-
continue
|
|
4477
|
-
|
|
4478
|
-
if edit == EDIT_PROP_SET:
|
|
4479
|
-
yield (permoffs, ('node', 'prop', 'set', f'{form}:{info[0]}'))
|
|
4480
|
-
continue
|
|
4481
|
-
|
|
4482
|
-
if edit == EDIT_PROP_DEL:
|
|
4483
|
-
yield (permoffs, ('node', 'prop', 'del', f'{form}:{info[0]}'))
|
|
4484
|
-
continue
|
|
4485
|
-
|
|
4486
|
-
if edit == EDIT_TAG_SET:
|
|
4487
|
-
if info[1] != (None, None):
|
|
4488
|
-
tagadds.append(info[0])
|
|
4489
|
-
yield (permoffs, ('node', 'tag', 'add', *info[0].split('.')))
|
|
4490
|
-
else:
|
|
4491
|
-
tags.append((len(info[0]), editoffs, info[0]))
|
|
4492
|
-
continue
|
|
4493
|
-
|
|
4494
|
-
if edit == EDIT_TAG_DEL:
|
|
4495
|
-
yield (permoffs, ('node', 'tag', 'del', *info[0].split('.')))
|
|
4496
|
-
continue
|
|
4497
|
-
|
|
4498
|
-
if edit == EDIT_TAGPROP_SET:
|
|
4499
|
-
yield (permoffs, ('node', 'tag', 'add', *info[0].split('.')))
|
|
4500
|
-
continue
|
|
4501
|
-
|
|
4502
|
-
if edit == EDIT_TAGPROP_DEL:
|
|
4503
|
-
yield (permoffs, ('node', 'tag', 'del', *info[0].split('.')))
|
|
4504
|
-
continue
|
|
4505
|
-
|
|
4506
|
-
if edit == EDIT_NODEDATA_SET:
|
|
4507
|
-
yield (permoffs, ('node', 'data', 'set', info[0]))
|
|
4508
|
-
continue
|
|
4509
|
-
|
|
4510
|
-
if edit == EDIT_NODEDATA_DEL:
|
|
4511
|
-
yield (permoffs, ('node', 'data', 'pop', info[0]))
|
|
4512
|
-
continue
|
|
4513
|
-
|
|
4514
|
-
if edit == EDIT_EDGE_ADD:
|
|
4515
|
-
yield (permoffs, ('node', 'edge', 'add', info[0]))
|
|
4516
|
-
continue
|
|
4517
|
-
|
|
4518
|
-
if edit == EDIT_EDGE_DEL:
|
|
4519
|
-
yield (permoffs, ('node', 'edge', 'del', info[0]))
|
|
4520
|
-
continue
|
|
4521
|
-
|
|
4522
|
-
for _, editoffs, tag in sorted(tags, reverse=True):
|
|
4523
|
-
look = tag + '.'
|
|
4524
|
-
if any([tagadd.startswith(look) for tagadd in tagadds]):
|
|
4525
|
-
continue
|
|
4526
|
-
|
|
4527
|
-
yield ((nodeoffs, editoffs), ('node', 'tag', 'add', *tag.split('.')))
|
|
4528
|
-
tagadds.append(tag)
|
synapse/lib/lmdbslab.py
CHANGED
|
@@ -1253,18 +1253,18 @@ class Slab(s_base.Base):
|
|
|
1253
1253
|
finally:
|
|
1254
1254
|
self._relXactForReading()
|
|
1255
1255
|
|
|
1256
|
-
def scanKeys(self, db=None):
|
|
1256
|
+
def scanKeys(self, db=None, nodup=False):
|
|
1257
1257
|
|
|
1258
|
-
with ScanKeys(self, db) as scan:
|
|
1258
|
+
with ScanKeys(self, db, nodup=nodup) as scan:
|
|
1259
1259
|
|
|
1260
1260
|
if not scan.first():
|
|
1261
1261
|
return
|
|
1262
1262
|
|
|
1263
1263
|
yield from scan.iternext()
|
|
1264
1264
|
|
|
1265
|
-
def scanKeysByPref(self, byts, db=None):
|
|
1265
|
+
def scanKeysByPref(self, byts, db=None, nodup=False):
|
|
1266
1266
|
|
|
1267
|
-
with ScanKeys(self, db) as scan:
|
|
1267
|
+
with ScanKeys(self, db, nodup=nodup) as scan:
|
|
1268
1268
|
|
|
1269
1269
|
if not scan.set_range(byts):
|
|
1270
1270
|
return
|
|
@@ -1283,7 +1283,7 @@ class Slab(s_base.Base):
|
|
|
1283
1283
|
'''
|
|
1284
1284
|
count = 0
|
|
1285
1285
|
size = len(byts)
|
|
1286
|
-
with ScanKeys(self, db) as scan:
|
|
1286
|
+
with ScanKeys(self, db, nodup=True) as scan:
|
|
1287
1287
|
|
|
1288
1288
|
if not scan.set_range(byts):
|
|
1289
1289
|
return 0
|
|
@@ -1293,7 +1293,7 @@ class Slab(s_base.Base):
|
|
|
1293
1293
|
if lkey[:size] != byts:
|
|
1294
1294
|
return count
|
|
1295
1295
|
|
|
1296
|
-
count +=
|
|
1296
|
+
count += scan.curs.count()
|
|
1297
1297
|
if maxsize is not None and maxsize == count:
|
|
1298
1298
|
return count
|
|
1299
1299
|
|
|
@@ -1756,14 +1756,21 @@ class ScanKeys(Scan):
|
|
|
1756
1756
|
An iterator over the keys of the database. If the database is dupsort, a key with multiple values with be yielded
|
|
1757
1757
|
once for each value.
|
|
1758
1758
|
'''
|
|
1759
|
+
def __init__(self, slab, db, nodup=False):
|
|
1760
|
+
Scan.__init__(self, slab, db)
|
|
1761
|
+
self.nodup = nodup
|
|
1762
|
+
|
|
1759
1763
|
def iterfunc(self):
|
|
1760
1764
|
if self.dupsort:
|
|
1761
|
-
|
|
1765
|
+
if self.nodup:
|
|
1766
|
+
return self.curs.iternext_nodup(keys=True, values=False)
|
|
1767
|
+
else:
|
|
1768
|
+
return Scan.iterfunc(self)
|
|
1762
1769
|
|
|
1763
1770
|
return self.curs.iternext(keys=True, values=False)
|
|
1764
1771
|
|
|
1765
1772
|
def resume(self):
|
|
1766
|
-
if self.dupsort:
|
|
1773
|
+
if self.dupsort and not self.nodup:
|
|
1767
1774
|
return Scan.resume(self)
|
|
1768
1775
|
|
|
1769
1776
|
return self.curs.set_range(self.atitem)
|
|
@@ -1772,13 +1779,13 @@ class ScanKeys(Scan):
|
|
|
1772
1779
|
'''
|
|
1773
1780
|
Returns if the cursor is at the value in atitem
|
|
1774
1781
|
'''
|
|
1775
|
-
if self.dupsort:
|
|
1782
|
+
if self.dupsort and not self.nodup:
|
|
1776
1783
|
return Scan.isatitem(self)
|
|
1777
1784
|
|
|
1778
1785
|
return self.atitem == self.curs.key()
|
|
1779
1786
|
|
|
1780
1787
|
def iternext(self):
|
|
1781
|
-
if self.dupsort:
|
|
1788
|
+
if self.dupsort and not self.nodup:
|
|
1782
1789
|
yield from (item[0] for item in Scan.iternext(self))
|
|
1783
1790
|
return
|
|
1784
1791
|
|
synapse/lib/node.py
CHANGED
|
@@ -270,7 +270,7 @@ class Node:
|
|
|
270
270
|
|
|
271
271
|
if self.form.isrunt:
|
|
272
272
|
if prop.info.get('ro'):
|
|
273
|
-
mesg = f'Cannot set read-only props on runt nodes: {repr(valu)
|
|
273
|
+
mesg = f'Cannot set read-only props on runt nodes: {s_common.trimText(repr(valu))}'
|
|
274
274
|
raise s_exc.IsRuntForm(mesg=mesg, form=self.form.full, prop=name)
|
|
275
275
|
|
|
276
276
|
await self.snap.core.runRuntPropSet(self, prop, valu)
|
synapse/lib/slabseqn.py
CHANGED
|
@@ -169,7 +169,7 @@ class SlabSeqn:
|
|
|
169
169
|
|
|
170
170
|
return s_common.int64un(byts) + 1
|
|
171
171
|
|
|
172
|
-
def iter(self, offs):
|
|
172
|
+
def iter(self, offs, reverse=False):
|
|
173
173
|
'''
|
|
174
174
|
Iterate over items in a sequence from a given offset.
|
|
175
175
|
|
|
@@ -180,10 +180,16 @@ class SlabSeqn:
|
|
|
180
180
|
(indx, valu): The index and valu of the item.
|
|
181
181
|
'''
|
|
182
182
|
startkey = s_common.int64en(offs)
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
183
|
+
if reverse:
|
|
184
|
+
for lkey, lval in self.slab.scanByRangeBack(startkey, db=self.db):
|
|
185
|
+
offs = s_common.int64un(lkey)
|
|
186
|
+
valu = s_msgpack.un(lval)
|
|
187
|
+
yield offs, valu
|
|
188
|
+
else:
|
|
189
|
+
for lkey, lval in self.slab.scanByRange(startkey, db=self.db):
|
|
190
|
+
offs = s_common.int64un(lkey)
|
|
191
|
+
valu = s_msgpack.un(lval)
|
|
192
|
+
yield offs, valu
|
|
187
193
|
|
|
188
194
|
async def aiter(self, offs, wait=False, timeout=None):
|
|
189
195
|
'''
|
synapse/lib/storm.py
CHANGED
|
@@ -2195,92 +2195,28 @@ class Runtime(s_base.Base):
|
|
|
2195
2195
|
return self.user.allowed(perms, gateiden=gateiden, default=default)
|
|
2196
2196
|
|
|
2197
2197
|
def allowedReason(self, perms, gateiden=None, default=None):
|
|
2198
|
-
'''
|
|
2199
|
-
Similar to allowed, but always prefer the default value specified by the caller.
|
|
2200
|
-
Default values are still pulled from permdefs if there is a match there; but still prefer caller default.
|
|
2201
|
-
This results in a ternary response that can be used to know if a rule had a positive/negative or no match.
|
|
2202
|
-
The matching reason metadata is also returned.
|
|
2203
|
-
'''
|
|
2204
2198
|
if self.asroot:
|
|
2205
2199
|
return self._admin_reason
|
|
2206
2200
|
|
|
2207
|
-
|
|
2208
|
-
permdef = self.snap.core.getPermDef(perms)
|
|
2209
|
-
if permdef:
|
|
2210
|
-
default = permdef.get('default', default)
|
|
2211
|
-
|
|
2212
|
-
return self.user.getAllowedReason(perms, gateiden=gateiden, default=default)
|
|
2201
|
+
return self.snap.core._propAllowedReason(self.user, perms, gateiden=gateiden, default=default)
|
|
2213
2202
|
|
|
2214
2203
|
def confirmPropSet(self, prop, layriden=None):
|
|
2204
|
+
if self.asroot:
|
|
2205
|
+
return
|
|
2215
2206
|
|
|
2216
2207
|
if layriden is None:
|
|
2217
2208
|
layriden = self.snap.wlyr.iden
|
|
2218
2209
|
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
if meta0.isadmin:
|
|
2222
|
-
return
|
|
2223
|
-
|
|
2224
|
-
allowed0 = meta0.value
|
|
2225
|
-
|
|
2226
|
-
meta1 = self.allowedReason(prop.setperms[1], gateiden=layriden)
|
|
2227
|
-
allowed1 = meta1.value
|
|
2228
|
-
|
|
2229
|
-
if allowed0:
|
|
2230
|
-
if allowed1:
|
|
2231
|
-
return
|
|
2232
|
-
elif allowed1 is False:
|
|
2233
|
-
# This is a allow-with-precedence case.
|
|
2234
|
-
# Inspect meta to determine if the rule a0 is more specific than rule a1
|
|
2235
|
-
if len(meta0.rule) >= len(meta1.rule):
|
|
2236
|
-
return
|
|
2237
|
-
self.user.raisePermDeny(prop.setperms[0], gateiden=layriden)
|
|
2238
|
-
return
|
|
2239
|
-
|
|
2240
|
-
if allowed1:
|
|
2241
|
-
if allowed0 is None:
|
|
2242
|
-
return
|
|
2243
|
-
# allowed0 here is False. This is a deny-with-precedence case.
|
|
2244
|
-
# Inspect meta to determine if the rule a1 is more specific than rule a0
|
|
2245
|
-
if len(meta1.rule) > len(meta0.rule):
|
|
2246
|
-
return
|
|
2247
|
-
|
|
2248
|
-
self.user.raisePermDeny(prop.setperms[0], gateiden=layriden)
|
|
2210
|
+
return self.snap.core.confirmPropSet(self.user, prop, layriden=layriden)
|
|
2249
2211
|
|
|
2250
2212
|
def confirmPropDel(self, prop, layriden=None):
|
|
2213
|
+
if self.asroot:
|
|
2214
|
+
return
|
|
2251
2215
|
|
|
2252
2216
|
if layriden is None:
|
|
2253
2217
|
layriden = self.snap.wlyr.iden
|
|
2254
2218
|
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
if meta0.isadmin:
|
|
2258
|
-
return
|
|
2259
|
-
|
|
2260
|
-
allowed0 = meta0.value
|
|
2261
|
-
meta1 = self.allowedReason(prop.delperms[1], gateiden=layriden)
|
|
2262
|
-
allowed1 = meta1.value
|
|
2263
|
-
|
|
2264
|
-
if allowed0:
|
|
2265
|
-
if allowed1:
|
|
2266
|
-
return
|
|
2267
|
-
elif allowed1 is False:
|
|
2268
|
-
# This is a allow-with-precedence case.
|
|
2269
|
-
# Inspect meta to determine if the rule a0 is more specific than rule a1
|
|
2270
|
-
if len(meta0.rule) >= len(meta1.rule):
|
|
2271
|
-
return
|
|
2272
|
-
self.user.raisePermDeny(prop.delperms[0], gateiden=layriden)
|
|
2273
|
-
return
|
|
2274
|
-
|
|
2275
|
-
if allowed1:
|
|
2276
|
-
if allowed0 is None:
|
|
2277
|
-
return
|
|
2278
|
-
# allowed0 here is False. This is a deny-with-precedence case.
|
|
2279
|
-
# Inspect meta to determine if the rule a1 is more specific than rule a0
|
|
2280
|
-
if len(meta1.rule) > len(meta0.rule):
|
|
2281
|
-
return
|
|
2282
|
-
|
|
2283
|
-
self.user.raisePermDeny(prop.delperms[0], gateiden=layriden)
|
|
2219
|
+
return self.snap.core.confirmPropDel(self.user, prop, layriden=layriden)
|
|
2284
2220
|
|
|
2285
2221
|
def confirmEasyPerm(self, item, perm, mesg=None):
|
|
2286
2222
|
if not self.asroot:
|
synapse/lib/stormhttp.py
CHANGED
|
@@ -549,7 +549,7 @@ class HttpResp(s_stormtypes.Prim):
|
|
|
549
549
|
return json.loads(valu.decode(encoding, errors))
|
|
550
550
|
|
|
551
551
|
except UnicodeDecodeError as e:
|
|
552
|
-
raise s_exc.StormRuntimeError(mesg=f'{e}: {repr(valu)
|
|
552
|
+
raise s_exc.StormRuntimeError(mesg=f'{e}: {s_common.trimText(repr(valu))}') from None
|
|
553
553
|
|
|
554
554
|
except json.JSONDecodeError as e:
|
|
555
555
|
mesg = f'Unable to decode HTTP response as json: {e.args[0]}'
|
synapse/lib/stormlib/auth.py
CHANGED
|
@@ -959,6 +959,18 @@ class User(s_stormtypes.Prim):
|
|
|
959
959
|
{'name': 'locked', 'type': 'boolean', 'desc': 'True to lock the user, false to unlock them.', },
|
|
960
960
|
),
|
|
961
961
|
'returns': {'type': 'null', }}},
|
|
962
|
+
{'name': 'setArchived', 'desc': '''
|
|
963
|
+
Set the archived status for a user.
|
|
964
|
+
|
|
965
|
+
Notes:
|
|
966
|
+
Setting a user as "archived" will also lock the user.
|
|
967
|
+
Removing a users "archived" status will not unlock the user.
|
|
968
|
+
''',
|
|
969
|
+
'type': {'type': 'function', '_funcname': '_methUserSetArchived',
|
|
970
|
+
'args': (
|
|
971
|
+
{'name': 'archived', 'type': 'boolean', 'desc': 'True to archive the user, false to unarchive them.', },
|
|
972
|
+
),
|
|
973
|
+
'returns': {'type': 'null', }}},
|
|
962
974
|
{'name': 'setPasswd', 'desc': 'Set the Users password.',
|
|
963
975
|
'type': {'type': 'function', '_funcname': '_methUserSetPasswd',
|
|
964
976
|
'args': (
|
|
@@ -1113,6 +1125,7 @@ class User(s_stormtypes.Prim):
|
|
|
1113
1125
|
'setEmail': self._methUserSetEmail,
|
|
1114
1126
|
'setLocked': self._methUserSetLocked,
|
|
1115
1127
|
'setPasswd': self._methUserSetPasswd,
|
|
1128
|
+
'setArchived': self._methUserSetArchived,
|
|
1116
1129
|
'getAllowedReason': self._methGetAllowedReason,
|
|
1117
1130
|
'genApiKey': self._methGenApiKey,
|
|
1118
1131
|
'getApiKey': self._methGetApiKey,
|
|
@@ -1286,6 +1299,10 @@ class User(s_stormtypes.Prim):
|
|
|
1286
1299
|
self.runt.confirm(('auth', 'user', 'set', 'locked'))
|
|
1287
1300
|
await self.runt.snap.core.setUserLocked(self.valu, await s_stormtypes.tobool(locked))
|
|
1288
1301
|
|
|
1302
|
+
async def _methUserSetArchived(self, archived):
|
|
1303
|
+
self.runt.confirm(('auth', 'user', 'set', 'archived'))
|
|
1304
|
+
await self.runt.snap.core.setUserArchived(self.valu, await s_stormtypes.tobool(archived))
|
|
1305
|
+
|
|
1289
1306
|
async def _methGenApiKey(self, name, duration=None):
|
|
1290
1307
|
name = await s_stormtypes.tostr(name)
|
|
1291
1308
|
duration = await s_stormtypes.toint(duration, noneok=True)
|
|
@@ -1703,6 +1720,8 @@ class LibUsers(s_stormtypes.Lib):
|
|
|
1703
1720
|
'desc': 'Controls changing a user\'s email address.'},
|
|
1704
1721
|
{'perm': ('auth', 'user', 'set', 'locked'), 'gate': 'cortex',
|
|
1705
1722
|
'desc': 'Controls locking/unlocking a user account.'},
|
|
1723
|
+
{'perm': ('auth', 'user', 'set', 'archived'), 'gate': 'cortex',
|
|
1724
|
+
'desc': 'Controls archiving/unarchiving a user account.'},
|
|
1706
1725
|
{'perm': ('auth', 'user', 'set', 'passwd'), 'gate': 'cortex',
|
|
1707
1726
|
'desc': 'Controls changing a user password.'},
|
|
1708
1727
|
{'perm': ('auth', 'user', 'set', 'rules'), 'gate': 'cortex',
|
synapse/lib/stormlib/cell.py
CHANGED
|
@@ -2,11 +2,16 @@ import asyncio
|
|
|
2
2
|
import logging
|
|
3
3
|
|
|
4
4
|
import synapse.exc as s_exc
|
|
5
|
-
import synapse.lib.
|
|
5
|
+
import synapse.lib.autodoc as s_autodoc
|
|
6
6
|
import synapse.lib.stormtypes as s_stormtypes
|
|
7
7
|
|
|
8
8
|
logger = logging.getLogger(__name__)
|
|
9
9
|
|
|
10
|
+
def prepHotfixDesc(txt):
|
|
11
|
+
lines = txt.split('\n')
|
|
12
|
+
lines = s_autodoc.scrubLines(lines)
|
|
13
|
+
lines = s_autodoc.ljuster(lines)
|
|
14
|
+
return lines
|
|
10
15
|
|
|
11
16
|
storm_missing_autoadds = '''
|
|
12
17
|
$absoluteOrder = $lib.view.list(deporder=$lib.true)
|
|
@@ -64,6 +69,17 @@ for $view in $views {
|
|
|
64
69
|
}
|
|
65
70
|
'''
|
|
66
71
|
|
|
72
|
+
storm_migrate_riskhasvuln = '''
|
|
73
|
+
for $view in $lib.view.list(deporder=$lib.true) {
|
|
74
|
+
view.exec $view.iden {
|
|
75
|
+
$layer = $lib.layer.get()
|
|
76
|
+
for ($buid, $sode) in $layer.getStorNodesByForm(risk:hasvuln) {
|
|
77
|
+
yield $buid
|
|
78
|
+
$lib.model.migration.s.riskHasVulnToVulnerable($node)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
'''
|
|
67
83
|
|
|
68
84
|
hotfixes = (
|
|
69
85
|
((1, 0, 0), {
|
|
@@ -78,6 +94,20 @@ hotfixes = (
|
|
|
78
94
|
'desc': 'Populate it:sec:cpe:v2_2 properties from existing CPE where the property is not set.',
|
|
79
95
|
'query': storm_missing_cpe22,
|
|
80
96
|
}),
|
|
97
|
+
((4, 0, 0), {
|
|
98
|
+
'desc': '''
|
|
99
|
+
Create risk:vulnerable nodes from existing risk:hasvuln nodes.
|
|
100
|
+
|
|
101
|
+
This hotfix should only be applied after all logic that would create
|
|
102
|
+
risk:hasvuln nodes has been updated. The hotfix uses the
|
|
103
|
+
$lib.model.migration.s.riskHasVulnToVulnerable() function,
|
|
104
|
+
which can be used directly for testing.
|
|
105
|
+
|
|
106
|
+
Tags, tag properties, edges, and node data will all be copied
|
|
107
|
+
to the risk:vulnerable nodes.
|
|
108
|
+
''',
|
|
109
|
+
'query': storm_migrate_riskhasvuln,
|
|
110
|
+
}),
|
|
81
111
|
)
|
|
82
112
|
runtime_fixes_key = 'cortex:runtime:stormfixes'
|
|
83
113
|
|
|
@@ -174,7 +204,9 @@ class CellLib(s_stormtypes.Lib):
|
|
|
174
204
|
assert desc is not None
|
|
175
205
|
assert vars is not None
|
|
176
206
|
|
|
177
|
-
|
|
207
|
+
title = prepHotfixDesc(desc)[0]
|
|
208
|
+
await self.runt.printf(f'Applying hotfix {vers} for [{title}]')
|
|
209
|
+
|
|
178
210
|
try:
|
|
179
211
|
query = await self.runt.getStormQuery(text)
|
|
180
212
|
async with self.runt.getSubRuntime(query, opts={'vars': vars}) as runt:
|
|
@@ -206,8 +238,14 @@ class CellLib(s_stormtypes.Lib):
|
|
|
206
238
|
continue
|
|
207
239
|
|
|
208
240
|
dowork = True
|
|
209
|
-
|
|
210
|
-
|
|
241
|
+
|
|
242
|
+
desclines = prepHotfixDesc(info.get('desc'))
|
|
243
|
+
await self.runt.printf(f'Would apply fix {vers} for [{desclines[0]}]')
|
|
244
|
+
if len(desclines) > 1:
|
|
245
|
+
for line in desclines[1:]:
|
|
246
|
+
await self.runt.printf(f' {line}' if line else '')
|
|
247
|
+
else:
|
|
248
|
+
await self.runt.printf('')
|
|
211
249
|
|
|
212
250
|
return dowork
|
|
213
251
|
|