synapse 2.199.0__py311-none-any.whl → 2.201.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +0 -6
- synapse/datamodel.py +10 -4
- synapse/lib/agenda.py +24 -5
- synapse/lib/ast.py +7 -10
- synapse/lib/hive.py +1 -265
- synapse/lib/schemas.py +1 -0
- synapse/lib/snap.py +2 -8
- synapse/lib/storm.py +20 -10
- synapse/lib/version.py +2 -2
- synapse/models/dns.py +1 -1
- synapse/models/economic.py +23 -23
- synapse/models/files.py +2 -2
- synapse/models/inet.py +2 -2
- synapse/models/infotech.py +7 -7
- synapse/models/person.py +1 -1
- synapse/models/proj.py +3 -2
- synapse/models/risk.py +1 -1
- synapse/models/transport.py +3 -3
- synapse/tests/test_cortex.py +3 -6
- synapse/tests/test_datamodel.py +2 -0
- synapse/tests/test_lib_agenda.py +41 -0
- synapse/tests/test_lib_ast.py +3 -0
- synapse/tests/test_lib_cell.py +4 -4
- synapse/tests/test_lib_hive.py +0 -38
- synapse/tests/test_lib_storm.py +11 -1
- synapse/tests/test_model_risk.py +4 -0
- synapse/tests/test_tools_autodoc.py +5 -0
- synapse/tests/test_tools_hiveload.py +19 -23
- synapse/tests/test_tools_hivesave.py +5 -7
- synapse/tests/utils.py +7 -26
- synapse/tools/autodoc.py +16 -1
- synapse/tools/hive/load.py +3 -9
- {synapse-2.199.0.dist-info → synapse-2.201.0.dist-info}/METADATA +2 -2
- {synapse-2.199.0.dist-info → synapse-2.201.0.dist-info}/RECORD +37 -38
- {synapse-2.199.0.dist-info → synapse-2.201.0.dist-info}/WHEEL +1 -1
- synapse/lib/hiveauth.py +0 -1336
- {synapse-2.199.0.dist-info → synapse-2.201.0.dist-info}/LICENSE +0 -0
- {synapse-2.199.0.dist-info → synapse-2.201.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py
CHANGED
|
@@ -20,7 +20,6 @@ import synapse.lib.base as s_base
|
|
|
20
20
|
import synapse.lib.cell as s_cell
|
|
21
21
|
import synapse.lib.chop as s_chop
|
|
22
22
|
import synapse.lib.coro as s_coro
|
|
23
|
-
import synapse.lib.hive as s_hive
|
|
24
23
|
import synapse.lib.view as s_view
|
|
25
24
|
import synapse.lib.cache as s_cache
|
|
26
25
|
import synapse.lib.const as s_const
|
|
@@ -855,7 +854,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
855
854
|
cellapi = CoreApi
|
|
856
855
|
viewapi = s_view.ViewApi
|
|
857
856
|
layerapi = s_layer.LayerApi
|
|
858
|
-
hiveapi = s_hive.HiveApi
|
|
859
857
|
|
|
860
858
|
viewctor = s_view.View.anit
|
|
861
859
|
layrctor = s_layer.Layer.anit
|
|
@@ -4656,10 +4654,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
4656
4654
|
if not path:
|
|
4657
4655
|
return await self.cellapi.anit(self, link, user)
|
|
4658
4656
|
|
|
4659
|
-
if path[0] == 'hive' and user.isAdmin():
|
|
4660
|
-
s_common.deprecated('Cortex /hive telepath path', curv='2.198.0', eolv='2.199.0')
|
|
4661
|
-
return await self.hiveapi.anit(self.hive, user)
|
|
4662
|
-
|
|
4663
4657
|
if path[0] == 'layer':
|
|
4664
4658
|
|
|
4665
4659
|
if len(path) == 1:
|
synapse/datamodel.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'''
|
|
2
2
|
An API to assist with the creation and enforcement of cortex data models.
|
|
3
3
|
'''
|
|
4
|
+
import sys
|
|
4
5
|
import asyncio
|
|
5
6
|
import logging
|
|
6
7
|
import collections
|
|
@@ -15,6 +16,7 @@ import synapse.lib.cache as s_cache
|
|
|
15
16
|
import synapse.lib.types as s_types
|
|
16
17
|
import synapse.lib.dyndeps as s_dyndeps
|
|
17
18
|
import synapse.lib.grammar as s_grammar
|
|
19
|
+
import synapse.lib.msgpack as s_msgpack
|
|
18
20
|
|
|
19
21
|
logger = logging.getLogger(__name__)
|
|
20
22
|
|
|
@@ -142,6 +144,9 @@ class Prop:
|
|
|
142
144
|
async def depfunc(node, oldv):
|
|
143
145
|
mesg = f'The property {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
|
|
144
146
|
await node.snap.warnonce(mesg)
|
|
147
|
+
if __debug__:
|
|
148
|
+
sys.audit('synapse.datamodel.Prop.deprecated', mesg, self.full)
|
|
149
|
+
|
|
145
150
|
self.onSet(depfunc)
|
|
146
151
|
|
|
147
152
|
def __repr__(self):
|
|
@@ -306,6 +311,8 @@ class Form:
|
|
|
306
311
|
async def depfunc(node):
|
|
307
312
|
mesg = f'The form {self.full} is deprecated or using a deprecated type and will be removed in 3.0.0'
|
|
308
313
|
await node.snap.warnonce(mesg)
|
|
314
|
+
if __debug__:
|
|
315
|
+
sys.audit('synapse.datamodel.Form.deprecated', mesg, self.full)
|
|
309
316
|
self.onAdd(depfunc)
|
|
310
317
|
|
|
311
318
|
def getStorNode(self, form):
|
|
@@ -1107,7 +1114,7 @@ class Model:
|
|
|
1107
1114
|
|
|
1108
1115
|
def _prepFormIface(self, form, iface):
|
|
1109
1116
|
|
|
1110
|
-
template = iface.get('template', {})
|
|
1117
|
+
template = s_msgpack.deepcopy(iface.get('template', {}))
|
|
1111
1118
|
template.update(form.type.info.get('template', {}))
|
|
1112
1119
|
|
|
1113
1120
|
def convert(item):
|
|
@@ -1153,10 +1160,9 @@ class Model:
|
|
|
1153
1160
|
for propname, typedef, propinfo in iface.get('props', ()):
|
|
1154
1161
|
|
|
1155
1162
|
# allow form props to take precedence
|
|
1156
|
-
if form.prop(propname) is
|
|
1157
|
-
|
|
1163
|
+
if (prop := form.prop(propname)) is None:
|
|
1164
|
+
prop = self._addFormProp(form, propname, typedef, propinfo)
|
|
1158
1165
|
|
|
1159
|
-
prop = self._addFormProp(form, propname, typedef, propinfo)
|
|
1160
1166
|
self.ifaceprops[f'{name}:{propname}'].append(prop.full)
|
|
1161
1167
|
|
|
1162
1168
|
if subifaces is not None:
|
synapse/lib/agenda.py
CHANGED
|
@@ -260,6 +260,7 @@ class _Appt:
|
|
|
260
260
|
'created',
|
|
261
261
|
'enabled',
|
|
262
262
|
'errcount',
|
|
263
|
+
'loglevel',
|
|
263
264
|
'nexttime',
|
|
264
265
|
'lasterrs',
|
|
265
266
|
'isrunning',
|
|
@@ -269,7 +270,7 @@ class _Appt:
|
|
|
269
270
|
'lastfinishtime',
|
|
270
271
|
}
|
|
271
272
|
|
|
272
|
-
def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False):
|
|
273
|
+
def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False, loglevel=None):
|
|
273
274
|
self.doc = ''
|
|
274
275
|
self.name = ''
|
|
275
276
|
self.task = None
|
|
@@ -284,6 +285,7 @@ class _Appt:
|
|
|
284
285
|
self._recidxnexttime = None # index of rec who is up next
|
|
285
286
|
self.view = view
|
|
286
287
|
self.created = created
|
|
288
|
+
self.loglevel = loglevel
|
|
287
289
|
|
|
288
290
|
if self.recur and not self.recs:
|
|
289
291
|
raise s_exc.BadTime(mesg='A recurrent appointment with no records')
|
|
@@ -364,7 +366,10 @@ class _Appt:
|
|
|
364
366
|
if val['ver'] != 1:
|
|
365
367
|
raise s_exc.BadStorageVersion(mesg=f"Found version {val['ver']}") # pragma: no cover
|
|
366
368
|
recs = [ApptRec.unpack(tupl) for tupl in val['recs']]
|
|
367
|
-
|
|
369
|
+
# TODO: MOAR INSANITY
|
|
370
|
+
loglevel = val.get('loglevel', 'WARNING')
|
|
371
|
+
appt = cls(stor, val['iden'], val['recur'], val['indx'], val['query'], val['creator'], recs,
|
|
372
|
+
nexttime=val['nexttime'], view=val.get('view'), loglevel=loglevel)
|
|
368
373
|
appt.doc = val.get('doc', '')
|
|
369
374
|
appt.name = val.get('name', '')
|
|
370
375
|
appt.pool = val.get('pool', False)
|
|
@@ -373,6 +378,7 @@ class _Appt:
|
|
|
373
378
|
appt.lastfinishtime = val['lastfinishtime']
|
|
374
379
|
appt.lastresult = val['lastresult']
|
|
375
380
|
appt.enabled = val['enabled']
|
|
381
|
+
appt.lasterrs = list(val.get('lasterrs', []))
|
|
376
382
|
|
|
377
383
|
return appt
|
|
378
384
|
|
|
@@ -422,8 +428,10 @@ class _Appt:
|
|
|
422
428
|
logger.warning('_Appt.edits() Invalid attribute received: %s = %r', name, valu, extra=extra)
|
|
423
429
|
continue
|
|
424
430
|
|
|
425
|
-
|
|
426
|
-
|
|
431
|
+
if name == 'lasterrs' and not isinstance(valu, list):
|
|
432
|
+
valu = list(valu)
|
|
433
|
+
|
|
434
|
+
setattr(self, name, valu)
|
|
427
435
|
|
|
428
436
|
await self.save()
|
|
429
437
|
|
|
@@ -559,6 +567,7 @@ class Agenda(s_base.Base):
|
|
|
559
567
|
creator = cdef.get('creator')
|
|
560
568
|
view = cdef.get('view')
|
|
561
569
|
created = cdef.get('created')
|
|
570
|
+
loglevel = cdef.get('loglevel', 'WARNING')
|
|
562
571
|
|
|
563
572
|
pool = cdef.get('pool', False)
|
|
564
573
|
|
|
@@ -603,7 +612,9 @@ class Agenda(s_base.Base):
|
|
|
603
612
|
incvals = (incvals, )
|
|
604
613
|
recs.extend(ApptRec(rd, incunit, v) for (rd, v) in itertools.product(reqdicts, incvals))
|
|
605
614
|
|
|
606
|
-
|
|
615
|
+
# TODO: this is insane. Make _Appt take the cdef directly...
|
|
616
|
+
appt = _Appt(self, iden, recur, indx, query, creator, recs, nexttime=nexttime, view=view,
|
|
617
|
+
created=created, pool=pool, loglevel=loglevel)
|
|
607
618
|
self._addappt(iden, appt)
|
|
608
619
|
|
|
609
620
|
appt.doc = cdef.get('doc', '')
|
|
@@ -841,7 +852,10 @@ class Agenda(s_base.Base):
|
|
|
841
852
|
extra={'synapse': {'iden': appt.iden, 'name': appt.name, 'user': user.iden, 'text': appt.query,
|
|
842
853
|
'username': user.name, 'view': appt.view}})
|
|
843
854
|
starttime = self._getNowTick()
|
|
855
|
+
|
|
844
856
|
success = False
|
|
857
|
+
loglevel = s_common.normLogLevel(appt.loglevel)
|
|
858
|
+
|
|
845
859
|
try:
|
|
846
860
|
opts = {
|
|
847
861
|
'user': user.iden,
|
|
@@ -861,6 +875,11 @@ class Agenda(s_base.Base):
|
|
|
861
875
|
if mesg[0] == 'node':
|
|
862
876
|
count += 1
|
|
863
877
|
|
|
878
|
+
elif mesg[0] == 'warn' and loglevel <= logging.WARNING:
|
|
879
|
+
text = mesg[1].get('mesg', '<missing message>')
|
|
880
|
+
extra = await self.core.getLogExtra(cron=appt.iden, **mesg[1])
|
|
881
|
+
logger.warning(f'Cron job {appt.iden} issued warning: {text}', extra=extra)
|
|
882
|
+
|
|
864
883
|
elif mesg[0] == 'err':
|
|
865
884
|
excname, errinfo = mesg[1]
|
|
866
885
|
errinfo.pop('eline', None)
|
synapse/lib/ast.py
CHANGED
|
@@ -217,13 +217,10 @@ class Query(AstNode):
|
|
|
217
217
|
genr = await stack.enter_async_context(contextlib.aclosing(oper.run(runt, genr)))
|
|
218
218
|
|
|
219
219
|
async for node, path in genr:
|
|
220
|
-
runt.tick()
|
|
221
220
|
yield node, path
|
|
222
221
|
|
|
223
222
|
async def iterNodePaths(self, runt, genr=None):
|
|
224
223
|
|
|
225
|
-
count = 0
|
|
226
|
-
|
|
227
224
|
self.optimize()
|
|
228
225
|
self.validate(runt)
|
|
229
226
|
|
|
@@ -231,18 +228,18 @@ class Query(AstNode):
|
|
|
231
228
|
if genr is None:
|
|
232
229
|
genr = runt.getInput()
|
|
233
230
|
|
|
231
|
+
count = 0
|
|
232
|
+
limit = runt.getOpt('limit')
|
|
233
|
+
|
|
234
234
|
async with contextlib.aclosing(self.run(runt, genr)) as agen:
|
|
235
235
|
async for node, path in agen:
|
|
236
236
|
|
|
237
|
-
runt.tick()
|
|
238
|
-
|
|
239
237
|
yield node, path
|
|
240
238
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
break
|
|
239
|
+
if limit is not None:
|
|
240
|
+
count += 1
|
|
241
|
+
if count >= limit:
|
|
242
|
+
break
|
|
246
243
|
|
|
247
244
|
class Lookup(Query):
|
|
248
245
|
'''
|
synapse/lib/hive.py
CHANGED
|
@@ -86,7 +86,7 @@ class Node(s_base.Base):
|
|
|
86
86
|
for name, node in self.kids.items():
|
|
87
87
|
yield name, node
|
|
88
88
|
|
|
89
|
-
class Hive(s_nexus.Pusher
|
|
89
|
+
class Hive(s_nexus.Pusher):
|
|
90
90
|
'''
|
|
91
91
|
An optionally persistent atomically accessed tree which implements
|
|
92
92
|
primitives for use in making distributed/clustered services.
|
|
@@ -95,8 +95,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
|
|
|
95
95
|
|
|
96
96
|
await s_nexus.Pusher.__anit__(self, 'hive', nexsroot=nexsroot)
|
|
97
97
|
|
|
98
|
-
s_telepath.Aware.__init__(self)
|
|
99
|
-
|
|
100
98
|
if conf is None:
|
|
101
99
|
conf = {}
|
|
102
100
|
|
|
@@ -161,24 +159,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
|
|
|
161
159
|
for cullname in culls:
|
|
162
160
|
await node.pop((cullname,))
|
|
163
161
|
|
|
164
|
-
async def getHiveAuth(self):
|
|
165
|
-
'''
|
|
166
|
-
Retrieve a HiveAuth for hive standalone or non-cell uses.
|
|
167
|
-
|
|
168
|
-
Note:
|
|
169
|
-
This is for the hive's own auth, or for non-cell auth. It isn't the same auth as for a cell
|
|
170
|
-
'''
|
|
171
|
-
import synapse.lib.hiveauth as s_hiveauth
|
|
172
|
-
if self.auth is None:
|
|
173
|
-
|
|
174
|
-
path = tuple(self.conf.get('auth:path').split('/'))
|
|
175
|
-
|
|
176
|
-
node = await self.open(path)
|
|
177
|
-
self.auth = await s_hiveauth.Auth.anit(node, nexsroot=self.nexsroot)
|
|
178
|
-
self.onfini(self.auth.fini)
|
|
179
|
-
|
|
180
|
-
return self.auth
|
|
181
|
-
|
|
182
162
|
async def _onHiveFini(self):
|
|
183
163
|
await self.root.fini()
|
|
184
164
|
|
|
@@ -423,27 +403,6 @@ class Hive(s_nexus.Pusher, s_telepath.Aware):
|
|
|
423
403
|
|
|
424
404
|
return node.valu
|
|
425
405
|
|
|
426
|
-
async def getTeleApi(self, link, mesg, path):
|
|
427
|
-
s_common.deprecated('Hive.getTeleApi', curv='2.198.0', eolv='2.199.0')
|
|
428
|
-
auth = await self.getHiveAuth()
|
|
429
|
-
|
|
430
|
-
if not self.conf.get('auth:en'):
|
|
431
|
-
user = await auth.getUserByName('root')
|
|
432
|
-
return await HiveApi.anit(self, user)
|
|
433
|
-
|
|
434
|
-
name, info = mesg[1].get('auth')
|
|
435
|
-
|
|
436
|
-
user = await auth.getUserByName(name)
|
|
437
|
-
if user is None:
|
|
438
|
-
raise s_exc.NoSuchUser(name=name)
|
|
439
|
-
|
|
440
|
-
# passwd None always fails...
|
|
441
|
-
passwd = info.get('passwd')
|
|
442
|
-
if not await user.tryPasswd(passwd):
|
|
443
|
-
raise s_exc.AuthDeny(mesg='Invalid password', user=user.iden, username=user.name)
|
|
444
|
-
|
|
445
|
-
return await HiveApi.anit(self, user)
|
|
446
|
-
|
|
447
406
|
async def _storLoadHive(self):
|
|
448
407
|
pass
|
|
449
408
|
|
|
@@ -480,224 +439,6 @@ class SlabHive(Hive):
|
|
|
480
439
|
lkey = '\x00'.join(full).encode('utf8')
|
|
481
440
|
self.slab.pop(lkey, db=self.db)
|
|
482
441
|
|
|
483
|
-
class HiveApi(s_base.Base):
|
|
484
|
-
|
|
485
|
-
async def __anit__(self, hive, user):
|
|
486
|
-
|
|
487
|
-
await s_base.Base.__anit__(self)
|
|
488
|
-
|
|
489
|
-
self.hive = hive
|
|
490
|
-
self.user = user
|
|
491
|
-
|
|
492
|
-
self.msgq = asyncio.Queue(maxsize=10000)
|
|
493
|
-
|
|
494
|
-
self.onfini(self._onHapiFini)
|
|
495
|
-
|
|
496
|
-
async def loadHiveTree(self, tree, path=(), trim=False):
|
|
497
|
-
s_common.deprecated('HiveApi.loadHiveTree', curv='2.167.0')
|
|
498
|
-
return await self.hive.loadHiveTree(tree, path=path, trim=trim)
|
|
499
|
-
|
|
500
|
-
async def saveHiveTree(self, path=()):
|
|
501
|
-
s_common.deprecated('HiveApi.saveHiveTree', curv='2.167.0')
|
|
502
|
-
return await self.hive.saveHiveTree(path=path)
|
|
503
|
-
|
|
504
|
-
async def treeAndSync(self, path, iden):
|
|
505
|
-
s_common.deprecated('HiveApi.treeAndSync', curv='2.167.0')
|
|
506
|
-
|
|
507
|
-
node = await self.hive.open(path)
|
|
508
|
-
|
|
509
|
-
# register handlers...
|
|
510
|
-
node.on('hive:add', self._onHiveEdit, base=self)
|
|
511
|
-
node.on('hive:set', self._onHiveEdit, base=self)
|
|
512
|
-
node.on('hive:pop', self._onHiveEdit, base=self)
|
|
513
|
-
|
|
514
|
-
# serialize the subtree into a message and return
|
|
515
|
-
# via the mesg queue so there is no get/update race
|
|
516
|
-
root = (node.valu, {})
|
|
517
|
-
|
|
518
|
-
todo = collections.deque([(node, root)])
|
|
519
|
-
|
|
520
|
-
# breadth first generator
|
|
521
|
-
while todo:
|
|
522
|
-
|
|
523
|
-
node, pode = todo.popleft()
|
|
524
|
-
|
|
525
|
-
for name, kidn in node.kids.items():
|
|
526
|
-
|
|
527
|
-
kidp = (kidn.valu, {})
|
|
528
|
-
pode[1][name] = kidp
|
|
529
|
-
|
|
530
|
-
todo.append((kidn, kidp))
|
|
531
|
-
|
|
532
|
-
await self.msgq.put(('hive:tree', {'path': path, 'tree': root}))
|
|
533
|
-
await self.msgq.put(('hive:sync', {'iden': iden}))
|
|
534
|
-
return
|
|
535
|
-
|
|
536
|
-
async def setAndSync(self, path, valu, iden, nexs=False):
|
|
537
|
-
s_common.deprecated('HiveApi.setAndSync', curv='2.167.0')
|
|
538
|
-
|
|
539
|
-
valu = await self.hive.set(path, valu, nexs=nexs)
|
|
540
|
-
await self.msgq.put(('hive:sync', {'iden': iden}))
|
|
541
|
-
return valu
|
|
542
|
-
|
|
543
|
-
async def addAndSync(self, path, valu, iden):
|
|
544
|
-
s_common.deprecated('HiveApi.addAndSync', curv='2.167.0')
|
|
545
|
-
|
|
546
|
-
valu = await self.hive.add(path, valu)
|
|
547
|
-
await self.msgq.put(('hive:sync', {'iden': iden}))
|
|
548
|
-
return valu
|
|
549
|
-
|
|
550
|
-
async def popAndSync(self, path, iden, nexs=False):
|
|
551
|
-
s_common.deprecated('HiveApi.popAndSync', curv='2.167.0')
|
|
552
|
-
|
|
553
|
-
valu = await self.hive.pop(path, nexs=nexs)
|
|
554
|
-
await self.msgq.put(('hive:sync', {'iden': iden}))
|
|
555
|
-
return valu
|
|
556
|
-
|
|
557
|
-
async def _onHapiFini(self):
|
|
558
|
-
await self.msgq.put(None)
|
|
559
|
-
|
|
560
|
-
async def _onHiveEdit(self, mesg):
|
|
561
|
-
self.msgq.put_nowait(mesg)
|
|
562
|
-
|
|
563
|
-
async def get(self, full):
|
|
564
|
-
s_common.deprecated('HiveApi.get', curv='2.167.0')
|
|
565
|
-
return await self.hive.get(full)
|
|
566
|
-
|
|
567
|
-
async def edits(self):
|
|
568
|
-
s_common.deprecated('HiveApi.edits', curv='2.167.0')
|
|
569
|
-
|
|
570
|
-
while not self.isfini:
|
|
571
|
-
|
|
572
|
-
item = await self.msgq.get()
|
|
573
|
-
if item is None:
|
|
574
|
-
return
|
|
575
|
-
|
|
576
|
-
yield item
|
|
577
|
-
|
|
578
|
-
class TeleHive(Hive):
|
|
579
|
-
'''
|
|
580
|
-
A Hive that acts as a consistent read cache for a telepath proxy Hive
|
|
581
|
-
'''
|
|
582
|
-
|
|
583
|
-
async def __anit__(self, proxy):
|
|
584
|
-
|
|
585
|
-
self.proxy = proxy
|
|
586
|
-
|
|
587
|
-
await Hive.__anit__(self)
|
|
588
|
-
|
|
589
|
-
self.lock = asyncio.Lock()
|
|
590
|
-
|
|
591
|
-
self.syncevents = {} # iden: asyncio.Event()
|
|
592
|
-
|
|
593
|
-
# fire a task to sync the sections of the tree we open
|
|
594
|
-
self.schedCoro(self._runHiveLoop())
|
|
595
|
-
|
|
596
|
-
self.mesgbus = await s_base.Base.anit()
|
|
597
|
-
self.mesgbus.on('hive:set', self._onHiveSet)
|
|
598
|
-
self.mesgbus.on('hive:pop', self._onHivePop)
|
|
599
|
-
self.mesgbus.on('hive:tree', self._onHiveTree)
|
|
600
|
-
self.mesgbus.on('hive:sync', self._onHiveSync)
|
|
601
|
-
|
|
602
|
-
self.onfini(self.mesgbus.fini)
|
|
603
|
-
|
|
604
|
-
self.onfini(proxy.fini)
|
|
605
|
-
|
|
606
|
-
async def _onHiveSync(self, mesg):
|
|
607
|
-
|
|
608
|
-
iden = mesg[1].get('iden')
|
|
609
|
-
evnt = self.syncevents.pop(iden, None)
|
|
610
|
-
if evnt is None:
|
|
611
|
-
return
|
|
612
|
-
|
|
613
|
-
evnt.set()
|
|
614
|
-
|
|
615
|
-
def _getSyncIden(self):
|
|
616
|
-
iden = s_common.guid()
|
|
617
|
-
evnt = asyncio.Event()
|
|
618
|
-
self.syncevents[iden] = evnt
|
|
619
|
-
return iden, evnt
|
|
620
|
-
|
|
621
|
-
async def _runHiveLoop(self):
|
|
622
|
-
while not self.isfini:
|
|
623
|
-
async for mesg in self.proxy.edits():
|
|
624
|
-
await self.mesgbus.dist(mesg)
|
|
625
|
-
|
|
626
|
-
async def _onHiveSet(self, mesg):
|
|
627
|
-
path = mesg[1].get('path')
|
|
628
|
-
valu = mesg[1].get('valu')
|
|
629
|
-
await Hive.set(self, path, valu)
|
|
630
|
-
|
|
631
|
-
async def _onHivePop(self, mesg):
|
|
632
|
-
path = mesg[1].get('path')
|
|
633
|
-
await Hive.pop(self, path)
|
|
634
|
-
|
|
635
|
-
async def _onHiveTree(self, mesg):
|
|
636
|
-
|
|
637
|
-
# get an entire tree update at once
|
|
638
|
-
path = mesg[1].get('path')
|
|
639
|
-
tree = mesg[1].get('tree')
|
|
640
|
-
|
|
641
|
-
node = await Hive.open(self, path)
|
|
642
|
-
|
|
643
|
-
todo = collections.deque([(node, path, tree)])
|
|
644
|
-
|
|
645
|
-
while todo:
|
|
646
|
-
|
|
647
|
-
node, path, (valu, kids) = todo.popleft()
|
|
648
|
-
|
|
649
|
-
# do *not* go through the set() API
|
|
650
|
-
node.valu = valu
|
|
651
|
-
for name, kidt in kids.items():
|
|
652
|
-
|
|
653
|
-
kidp = path + (name,)
|
|
654
|
-
kidn = await Hive.open(self, kidp)
|
|
655
|
-
|
|
656
|
-
todo.append((kidn, kidp, kidt))
|
|
657
|
-
|
|
658
|
-
async def set(self, path, valu, nexs=False):
|
|
659
|
-
iden, evnt = self._getSyncIden()
|
|
660
|
-
valu = await self.proxy.setAndSync(path, valu, iden, nexs=nexs)
|
|
661
|
-
await evnt.wait()
|
|
662
|
-
return valu
|
|
663
|
-
|
|
664
|
-
async def add(self, path, valu):
|
|
665
|
-
iden, evnt = self._getSyncIden()
|
|
666
|
-
valu = await self.proxy.addAndSync(path, valu, iden)
|
|
667
|
-
await evnt.wait()
|
|
668
|
-
return valu
|
|
669
|
-
|
|
670
|
-
async def pop(self, path, nexs=False):
|
|
671
|
-
iden, evnt = self._getSyncIden()
|
|
672
|
-
valu = await self.proxy.popAndSync(path, iden, nexs=nexs)
|
|
673
|
-
await evnt.wait()
|
|
674
|
-
return valu
|
|
675
|
-
|
|
676
|
-
async def get(self, path):
|
|
677
|
-
return await self.proxy.get(path)
|
|
678
|
-
|
|
679
|
-
async def open(self, path):
|
|
680
|
-
|
|
681
|
-
# try once pre-lock for speed
|
|
682
|
-
node = self.nodes.get(path)
|
|
683
|
-
if node is not None:
|
|
684
|
-
return node
|
|
685
|
-
|
|
686
|
-
async with self.lock:
|
|
687
|
-
|
|
688
|
-
# try again with lock to avoid race
|
|
689
|
-
node = self.nodes.get(path)
|
|
690
|
-
if node is not None:
|
|
691
|
-
return node
|
|
692
|
-
|
|
693
|
-
iden, evnt = self._getSyncIden()
|
|
694
|
-
|
|
695
|
-
await self.proxy.treeAndSync(path, iden)
|
|
696
|
-
|
|
697
|
-
await evnt.wait()
|
|
698
|
-
|
|
699
|
-
return self.nodes.get(path)
|
|
700
|
-
|
|
701
442
|
class HiveDict(s_base.Base):
|
|
702
443
|
'''
|
|
703
444
|
'''
|
|
@@ -758,11 +499,6 @@ def iterpath(path):
|
|
|
758
499
|
for i in range(len(path)):
|
|
759
500
|
yield path[:i + 1]
|
|
760
501
|
|
|
761
|
-
async def openurl(url, **opts):
|
|
762
|
-
s_common.deprecated('synapse.lib.hive.openurl()', curv='2.198.0', eolv='2.199.0')
|
|
763
|
-
prox = await s_telepath.openurl(url, **opts)
|
|
764
|
-
return await TeleHive.anit(prox)
|
|
765
|
-
|
|
766
502
|
async def opendir(dirn, conf=None):
|
|
767
503
|
slab = await s_slab.Slab.anit(dirn, map_size=s_const.gibibyte)
|
|
768
504
|
db = slab.initdb('hive')
|
synapse/lib/schemas.py
CHANGED
synapse/lib/snap.py
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import sys
|
|
4
3
|
import types
|
|
5
4
|
import asyncio
|
|
6
5
|
import logging
|
|
@@ -12,14 +11,12 @@ import synapse.exc as s_exc
|
|
|
12
11
|
import synapse.common as s_common
|
|
13
12
|
|
|
14
13
|
import synapse.lib.base as s_base
|
|
15
|
-
import synapse.lib.coro as s_coro
|
|
16
14
|
import synapse.lib.node as s_node
|
|
17
15
|
import synapse.lib.time as s_time
|
|
18
16
|
import synapse.lib.cache as s_cache
|
|
19
17
|
import synapse.lib.layer as s_layer
|
|
20
18
|
import synapse.lib.storm as s_storm
|
|
21
19
|
import synapse.lib.types as s_types
|
|
22
|
-
import synapse.lib.spooled as s_spooled
|
|
23
20
|
|
|
24
21
|
logger = logging.getLogger(__name__)
|
|
25
22
|
|
|
@@ -1606,7 +1603,8 @@ class Snap(s_base.Base):
|
|
|
1606
1603
|
proptype = prop.type
|
|
1607
1604
|
for prop in prop.getAlts():
|
|
1608
1605
|
if prop.type.isarray and prop.type.arraytype == proptype:
|
|
1609
|
-
|
|
1606
|
+
arryvalu = node.get(prop.name)
|
|
1607
|
+
if arryvalu is not None and valu in arryvalu:
|
|
1610
1608
|
return True
|
|
1611
1609
|
else:
|
|
1612
1610
|
if node.get(prop.name) == valu:
|
|
@@ -1694,10 +1692,6 @@ class Snap(s_base.Base):
|
|
|
1694
1692
|
return tagnode
|
|
1695
1693
|
|
|
1696
1694
|
async def _raiseOnStrict(self, ctor, mesg, **info):
|
|
1697
|
-
if __debug__:
|
|
1698
|
-
if issubclass(ctor, s_exc.IsDeprLocked):
|
|
1699
|
-
sys.audit('synapse.exc.IsDeprLocked', (mesg, info))
|
|
1700
|
-
|
|
1701
1695
|
if self.strict:
|
|
1702
1696
|
raise ctor(mesg=mesg, **info)
|
|
1703
1697
|
await self.warn(mesg)
|
synapse/lib/storm.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import types
|
|
2
2
|
import pprint
|
|
3
3
|
import asyncio
|
|
4
|
-
import hashlib
|
|
5
4
|
import logging
|
|
6
5
|
import argparse
|
|
7
6
|
import contextlib
|
|
@@ -22,13 +21,10 @@ import synapse.lib.snap as s_snap
|
|
|
22
21
|
import synapse.lib.cache as s_cache
|
|
23
22
|
import synapse.lib.layer as s_layer
|
|
24
23
|
import synapse.lib.scope as s_scope
|
|
25
|
-
import synapse.lib.config as s_config
|
|
26
24
|
import synapse.lib.autodoc as s_autodoc
|
|
27
|
-
import synapse.lib.grammar as s_grammar
|
|
28
25
|
import synapse.lib.msgpack as s_msgpack
|
|
29
26
|
import synapse.lib.schemas as s_schemas
|
|
30
27
|
import synapse.lib.spooled as s_spooled
|
|
31
|
-
import synapse.lib.version as s_version
|
|
32
28
|
import synapse.lib.hashitem as s_hashitem
|
|
33
29
|
import synapse.lib.stormctrl as s_stormctrl
|
|
34
30
|
import synapse.lib.stormtypes as s_stormtypes
|
|
@@ -1726,9 +1722,6 @@ class Runtime(s_base.Base):
|
|
|
1726
1722
|
async def warnonce(self, mesg, **info):
|
|
1727
1723
|
return await self.snap.warnonce(mesg, **info)
|
|
1728
1724
|
|
|
1729
|
-
def tick(self):
|
|
1730
|
-
pass
|
|
1731
|
-
|
|
1732
1725
|
def cancel(self):
|
|
1733
1726
|
self.task.cancel()
|
|
1734
1727
|
|
|
@@ -1964,7 +1957,6 @@ class Runtime(s_base.Base):
|
|
|
1964
1957
|
nodegenr = subgraph.run(self, nodegenr)
|
|
1965
1958
|
|
|
1966
1959
|
async for item in nodegenr:
|
|
1967
|
-
self.tick()
|
|
1968
1960
|
yield item
|
|
1969
1961
|
|
|
1970
1962
|
except RecursionError:
|
|
@@ -2077,7 +2069,7 @@ class Runtime(s_base.Base):
|
|
|
2077
2069
|
|
|
2078
2070
|
class Parser:
|
|
2079
2071
|
|
|
2080
|
-
def __init__(self, prog=None, descr=None, root=None, model=None):
|
|
2072
|
+
def __init__(self, prog=None, descr=None, root=None, model=None, cdef=None):
|
|
2081
2073
|
|
|
2082
2074
|
if root is None:
|
|
2083
2075
|
root = self
|
|
@@ -2088,6 +2080,7 @@ class Parser:
|
|
|
2088
2080
|
|
|
2089
2081
|
self.prog = prog
|
|
2090
2082
|
self.descr = descr
|
|
2083
|
+
self.cdef = cdef
|
|
2091
2084
|
|
|
2092
2085
|
self.exc = None
|
|
2093
2086
|
|
|
@@ -2365,6 +2358,21 @@ class Parser:
|
|
|
2365
2358
|
|
|
2366
2359
|
self._printf(f'Usage: {self.prog} [options] {posargs}')
|
|
2367
2360
|
|
|
2361
|
+
if self.cdef is not None and (endpoints := self.cdef.get('endpoints')):
|
|
2362
|
+
self._printf('')
|
|
2363
|
+
self._printf('Endpoints:')
|
|
2364
|
+
self._printf('')
|
|
2365
|
+
base_w = 32
|
|
2366
|
+
wrap_w = 120 - base_w
|
|
2367
|
+
for endpoint in endpoints:
|
|
2368
|
+
path = endpoint['path']
|
|
2369
|
+
desc = endpoint.get('desc', '')
|
|
2370
|
+
base = f' {path}'
|
|
2371
|
+
wrap_desc = self._wrap_text(desc, wrap_w) if desc else ['']
|
|
2372
|
+
self._printf(f'{base:<{base_w-2}}: {wrap_desc[0]}')
|
|
2373
|
+
for ln in wrap_desc[1:]:
|
|
2374
|
+
self._printf(f'{"":<{base_w}}{ln}')
|
|
2375
|
+
|
|
2368
2376
|
options = [x for x in self.allargs if x[0][0].startswith('-')]
|
|
2369
2377
|
|
|
2370
2378
|
self._printf('')
|
|
@@ -2655,6 +2663,7 @@ class PureCmd(Cmd):
|
|
|
2655
2663
|
if inputs:
|
|
2656
2664
|
pars.set_inputs(inputs)
|
|
2657
2665
|
|
|
2666
|
+
pars.cdef = self.cdef
|
|
2658
2667
|
return pars
|
|
2659
2668
|
|
|
2660
2669
|
async def execStormCmd(self, runt, genr):
|
|
@@ -2830,7 +2839,8 @@ class BatchCmd(Cmd):
|
|
|
2830
2839
|
mesg = f'Specified batch size ({size}) is above the maximum (10000).'
|
|
2831
2840
|
raise s_exc.StormRuntimeError(mesg=mesg)
|
|
2832
2841
|
|
|
2833
|
-
|
|
2842
|
+
_query = await s_stormtypes.tostr(self.opts.query)
|
|
2843
|
+
query = await runt.getStormQuery(_query)
|
|
2834
2844
|
doyield = await s_stormtypes.tobool(self.opts.cond)
|
|
2835
2845
|
|
|
2836
2846
|
async with runt.getSubRuntime(query, opts={'vars': {'nodes': []}}) as subr:
|
synapse/lib/version.py
CHANGED
|
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
|
|
|
223
223
|
##############################################################################
|
|
224
224
|
# The following are touched during the release process by bumpversion.
|
|
225
225
|
# Do not modify these directly.
|
|
226
|
-
version = (2,
|
|
226
|
+
version = (2, 201, 0)
|
|
227
227
|
verstring = '.'.join([str(x) for x in version])
|
|
228
|
-
commit = '
|
|
228
|
+
commit = 'eed8c13ff316a77ae835539215c862d052d31994'
|
synapse/models/dns.py
CHANGED
|
@@ -147,7 +147,7 @@ class DnsModule(s_module.CoreModule):
|
|
|
147
147
|
|
|
148
148
|
('inet:dns:txt', ('comp', {'fields': (('fqdn', 'inet:fqdn'), ('txt', 'str'))}), {
|
|
149
149
|
'ex': '(hehe.vertex.link,"fancy TXT record")',
|
|
150
|
-
'doc': 'The result of a DNS
|
|
150
|
+
'doc': 'The result of a DNS TXT record lookup.'}),
|
|
151
151
|
|
|
152
152
|
('inet:dns:type', ('int', {}), {
|
|
153
153
|
'doc': 'A DNS query/answer type integer.'}),
|