synapse 2.173.1__py311-none-any.whl → 2.174.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +1 -1
- synapse/common.py +19 -5
- synapse/cortex.py +46 -10
- synapse/lib/agenda.py +6 -0
- synapse/lib/ast.py +1 -1
- synapse/lib/lmdbslab.py +11 -1
- synapse/lib/modelrev.py +17 -1
- synapse/lib/modules.py +1 -0
- synapse/lib/msgpack.py +25 -3
- synapse/lib/nexus.py +26 -22
- synapse/lib/schemas.py +31 -0
- synapse/lib/stormsvc.py +30 -11
- synapse/lib/stormtypes.py +23 -9
- synapse/lib/trigger.py +0 -4
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +2 -0
- synapse/models/crypto.py +22 -0
- synapse/models/economic.py +23 -2
- synapse/models/entity.py +16 -0
- synapse/models/files.py +4 -1
- synapse/models/geopol.py +3 -0
- synapse/models/orgs.py +3 -4
- synapse/tests/test_cortex.py +13 -0
- synapse/tests/test_lib_agenda.py +129 -1
- synapse/tests/test_lib_ast.py +21 -0
- synapse/tests/test_lib_grammar.py +4 -0
- synapse/tests/test_lib_httpapi.py +1 -0
- synapse/tests/test_lib_lmdbslab.py +16 -1
- synapse/tests/test_lib_modelrev.py +57 -0
- synapse/tests/test_lib_msgpack.py +58 -8
- synapse/tests/test_lib_nexus.py +44 -1
- synapse/tests/test_lib_storm.py +7 -7
- synapse/tests/test_lib_stormsvc.py +128 -51
- synapse/tests/test_lib_stormtypes.py +43 -4
- synapse/tests/test_lib_trigger.py +23 -4
- synapse/tests/test_model_crypto.py +6 -0
- synapse/tests/test_model_economic.py +14 -1
- synapse/tests/test_model_geopol.py +3 -0
- synapse/tools/changelog.py +236 -0
- {synapse-2.173.1.dist-info → synapse-2.174.0.dist-info}/METADATA +1 -1
- {synapse-2.173.1.dist-info → synapse-2.174.0.dist-info}/RECORD +44 -42
- {synapse-2.173.1.dist-info → synapse-2.174.0.dist-info}/WHEEL +1 -1
- {synapse-2.173.1.dist-info → synapse-2.174.0.dist-info}/LICENSE +0 -0
- {synapse-2.173.1.dist-info → synapse-2.174.0.dist-info}/top_level.txt +0 -0
synapse/axon.py
CHANGED
|
@@ -493,7 +493,7 @@ class AxonApi(s_cell.CellApi, s_share.Share): # type: ignore
|
|
|
493
493
|
|
|
494
494
|
async def wants(self, sha256s):
|
|
495
495
|
'''
|
|
496
|
-
Get a list of sha256 values the axon does not have from
|
|
496
|
+
Get a list of sha256 values the axon does not have from an input list.
|
|
497
497
|
|
|
498
498
|
Args:
|
|
499
499
|
sha256s (list): A list of sha256 values as bytes.
|
synapse/common.py
CHANGED
|
@@ -11,6 +11,7 @@ import types
|
|
|
11
11
|
import base64
|
|
12
12
|
import shutil
|
|
13
13
|
import struct
|
|
14
|
+
import typing
|
|
14
15
|
import asyncio
|
|
15
16
|
import decimal
|
|
16
17
|
import fnmatch
|
|
@@ -379,7 +380,7 @@ def reqbytes(*paths):
|
|
|
379
380
|
with reqfile(*paths) as fd:
|
|
380
381
|
return fd.read()
|
|
381
382
|
|
|
382
|
-
def genfile(*paths):
|
|
383
|
+
def genfile(*paths) -> typing.BinaryIO:
|
|
383
384
|
'''
|
|
384
385
|
Create or open (for read/write) a file path join.
|
|
385
386
|
|
|
@@ -396,7 +397,7 @@ def genfile(*paths):
|
|
|
396
397
|
to append.
|
|
397
398
|
|
|
398
399
|
Returns:
|
|
399
|
-
|
|
400
|
+
A file-object which can be read/written too.
|
|
400
401
|
'''
|
|
401
402
|
path = genpath(*paths)
|
|
402
403
|
gendir(os.path.dirname(path))
|
|
@@ -535,13 +536,26 @@ def yamlload(*paths):
|
|
|
535
536
|
with io.open(path, 'rb') as fd:
|
|
536
537
|
return yamlloads(fd)
|
|
537
538
|
|
|
539
|
+
def yamldump(obj, stream: typing.Optional[typing.BinaryIO] =None) -> bytes:
|
|
540
|
+
'''
|
|
541
|
+
Dump a object to yaml.
|
|
542
|
+
|
|
543
|
+
Args:
|
|
544
|
+
obj: The object to serialize.
|
|
545
|
+
stream: The optional stream to write the stream too.
|
|
546
|
+
|
|
547
|
+
Returns:
|
|
548
|
+
The raw yaml bytes if stream is not provided.
|
|
549
|
+
'''
|
|
550
|
+
return yaml.dump(obj, allow_unicode=True, default_flow_style=False,
|
|
551
|
+
default_style='', explicit_start=True, explicit_end=True,
|
|
552
|
+
encoding='utf8', stream=stream, Dumper=Dumper)
|
|
553
|
+
|
|
538
554
|
def yamlsave(obj, *paths):
|
|
539
555
|
path = genpath(*paths)
|
|
540
556
|
with genfile(path) as fd:
|
|
541
557
|
fd.truncate(0)
|
|
542
|
-
|
|
543
|
-
default_style='', explicit_start=True, explicit_end=True,
|
|
544
|
-
encoding='utf8', stream=fd, Dumper=Dumper)
|
|
558
|
+
yamldump(obj, stream=fd)
|
|
545
559
|
|
|
546
560
|
def yamlmod(obj, *paths):
|
|
547
561
|
'''
|
synapse/cortex.py
CHANGED
|
@@ -42,6 +42,7 @@ import synapse.lib.schemas as s_schemas
|
|
|
42
42
|
import synapse.lib.spooled as s_spooled
|
|
43
43
|
import synapse.lib.version as s_version
|
|
44
44
|
import synapse.lib.urlhelp as s_urlhelp
|
|
45
|
+
import synapse.lib.hashitem as s_hashitem
|
|
45
46
|
import synapse.lib.jsonstor as s_jsonstor
|
|
46
47
|
import synapse.lib.modelrev as s_modelrev
|
|
47
48
|
import synapse.lib.stormsvc as s_stormsvc
|
|
@@ -949,6 +950,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
949
950
|
|
|
950
951
|
# Initialize our storage and views
|
|
951
952
|
await self._initCoreAxon()
|
|
953
|
+
await self._initJsonStor()
|
|
952
954
|
|
|
953
955
|
await self._initCoreLayers()
|
|
954
956
|
await self._initCoreViews()
|
|
@@ -1009,8 +1011,16 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1009
1011
|
await self._bumpCellVers('cortex:storage', (
|
|
1010
1012
|
(1, self._storUpdateMacros),
|
|
1011
1013
|
(2, self._storLayrFeedDefaults),
|
|
1014
|
+
(3, self._updateTriggerViewIdens),
|
|
1012
1015
|
), nexs=False)
|
|
1013
1016
|
|
|
1017
|
+
async def _updateTriggerViewIdens(self):
|
|
1018
|
+
for view in self.views.values():
|
|
1019
|
+
for trigiden, trigger in await view.listTriggers():
|
|
1020
|
+
if trigger.get('view') != view.iden:
|
|
1021
|
+
trigger.tdef['view'] = view.iden
|
|
1022
|
+
await view.trigdict.set(trigiden, trigger.tdef)
|
|
1023
|
+
|
|
1014
1024
|
async def _viewNomergeToProtected(self):
|
|
1015
1025
|
for view in self.views.values():
|
|
1016
1026
|
nomerge = view.info.get('nomerge', False)
|
|
@@ -1462,8 +1472,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1462
1472
|
async def initServiceRuntime(self):
|
|
1463
1473
|
|
|
1464
1474
|
# do any post-nexus initialization here...
|
|
1465
|
-
await self._initJsonStor()
|
|
1466
|
-
|
|
1467
1475
|
if self.isactive:
|
|
1468
1476
|
await self._checkNexsIndx()
|
|
1469
1477
|
|
|
@@ -2550,7 +2558,10 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
2550
2558
|
name = pkgdef.get('name')
|
|
2551
2559
|
olddef = self.pkghive.get(name, None)
|
|
2552
2560
|
if olddef is not None:
|
|
2553
|
-
|
|
2561
|
+
if s_hashitem.hashitem(pkgdef) != s_hashitem.hashitem(olddef):
|
|
2562
|
+
await self._dropStormPkg(olddef)
|
|
2563
|
+
else:
|
|
2564
|
+
return
|
|
2554
2565
|
|
|
2555
2566
|
await self.loadStormPkg(pkgdef)
|
|
2556
2567
|
await self.pkghive.set(name, pkgdef)
|
|
@@ -2965,17 +2976,19 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
2965
2976
|
'''
|
|
2966
2977
|
Delete storm packages associated with a service.
|
|
2967
2978
|
'''
|
|
2968
|
-
|
|
2969
|
-
for _, pdef in self.pkghive.items():
|
|
2970
|
-
pkgiden = pdef.get('svciden')
|
|
2971
|
-
if pkgiden and pkgiden == iden:
|
|
2972
|
-
oldpkgs.append(pdef)
|
|
2973
|
-
|
|
2974
|
-
for pkg in oldpkgs:
|
|
2979
|
+
for pkg in self.getStormSvcPkgs(iden):
|
|
2975
2980
|
name = pkg.get('name')
|
|
2976
2981
|
if name:
|
|
2977
2982
|
await self._delStormPkg(name)
|
|
2978
2983
|
|
|
2984
|
+
def getStormSvcPkgs(self, iden):
|
|
2985
|
+
pkgs = []
|
|
2986
|
+
for _, pdef in self.pkghive.items():
|
|
2987
|
+
pkgiden = pdef.get('svciden')
|
|
2988
|
+
if pkgiden and pkgiden == iden:
|
|
2989
|
+
pkgs.append(pdef)
|
|
2990
|
+
return pkgs
|
|
2991
|
+
|
|
2979
2992
|
async def setStormSvcEvents(self, iden, edef):
|
|
2980
2993
|
'''
|
|
2981
2994
|
Set the event callbacks for a storm service. Extends the sdef dict.
|
|
@@ -6113,6 +6126,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
6113
6126
|
Args:
|
|
6114
6127
|
iden (bytes): The iden of the cron job to be deleted
|
|
6115
6128
|
'''
|
|
6129
|
+
await self._killCronTask(iden)
|
|
6116
6130
|
try:
|
|
6117
6131
|
await self.agenda.delete(iden)
|
|
6118
6132
|
except s_exc.NoSuchIden:
|
|
@@ -6142,6 +6156,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
6142
6156
|
'''
|
|
6143
6157
|
await self.agenda.enable(iden)
|
|
6144
6158
|
await self.feedBeholder('cron:enable', {'iden': iden}, gates=[iden])
|
|
6159
|
+
logger.info(f'Enabled cron job {iden}', extra=await self.getLogExtra(iden=iden, status='MODIFY'))
|
|
6145
6160
|
|
|
6146
6161
|
@s_nexus.Pusher.onPushAuto('cron:disable')
|
|
6147
6162
|
async def disableCronJob(self, iden):
|
|
@@ -6152,7 +6167,28 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
6152
6167
|
iden (bytes): The iden of the cron job to be changed
|
|
6153
6168
|
'''
|
|
6154
6169
|
await self.agenda.disable(iden)
|
|
6170
|
+
await self._killCronTask(iden)
|
|
6155
6171
|
await self.feedBeholder('cron:disable', {'iden': iden}, gates=[iden])
|
|
6172
|
+
logger.info(f'Disabled cron job {iden}', extra=await self.getLogExtra(iden=iden, status='MODIFY'))
|
|
6173
|
+
|
|
6174
|
+
async def killCronTask(self, iden):
|
|
6175
|
+
if self.agenda.appts.get(iden) is None:
|
|
6176
|
+
return False
|
|
6177
|
+
return await self._push('cron:task:kill', iden)
|
|
6178
|
+
|
|
6179
|
+
@s_nexus.Pusher.onPush('cron:task:kill')
|
|
6180
|
+
async def _killCronTask(self, iden):
|
|
6181
|
+
|
|
6182
|
+
appt = self.agenda.appts.get(iden)
|
|
6183
|
+
if appt is None:
|
|
6184
|
+
return False
|
|
6185
|
+
|
|
6186
|
+
task = appt.task
|
|
6187
|
+
if task is None:
|
|
6188
|
+
return False
|
|
6189
|
+
|
|
6190
|
+
self.schedCoro(task.kill())
|
|
6191
|
+
return True
|
|
6156
6192
|
|
|
6157
6193
|
async def listCronJobs(self):
|
|
6158
6194
|
'''
|
synapse/lib/agenda.py
CHANGED
|
@@ -274,6 +274,7 @@ class _Appt:
|
|
|
274
274
|
def __init__(self, stor, iden, recur, indx, query, creator, recs, nexttime=None, view=None, created=None, pool=False):
|
|
275
275
|
self.doc = ''
|
|
276
276
|
self.name = ''
|
|
277
|
+
self.task = None
|
|
277
278
|
self.stor = stor
|
|
278
279
|
self.pool = pool
|
|
279
280
|
self.iden = iden
|
|
@@ -801,7 +802,12 @@ class Agenda(s_base.Base):
|
|
|
801
802
|
|
|
802
803
|
coro = self._runJob(user, appt)
|
|
803
804
|
task = self.core.runActiveTask(coro)
|
|
805
|
+
|
|
804
806
|
appt.task = await self.core.boss.promotetask(task, f'Cron {appt.iden}', user, info=info)
|
|
807
|
+
async def fini():
|
|
808
|
+
appt.task = None
|
|
809
|
+
|
|
810
|
+
appt.task.onfini(fini)
|
|
805
811
|
|
|
806
812
|
async def _markfailed(self, appt, reason):
|
|
807
813
|
now = self._getNowTick()
|
synapse/lib/ast.py
CHANGED
|
@@ -3017,11 +3017,11 @@ class ArrayCond(Cond):
|
|
|
3017
3017
|
|
|
3018
3018
|
async def getCondEval(self, runt):
|
|
3019
3019
|
|
|
3020
|
-
name = await self.kids[0].compute(runt, None)
|
|
3021
3020
|
cmpr = await self.kids[1].compute(runt, None)
|
|
3022
3021
|
|
|
3023
3022
|
async def cond(node, path):
|
|
3024
3023
|
|
|
3024
|
+
name = await self.kids[0].compute(runt, None)
|
|
3025
3025
|
prop = node.form.props.get(name)
|
|
3026
3026
|
if prop is None:
|
|
3027
3027
|
raise self.kids[0].addExcInfo(s_exc.NoSuchProp.init(name))
|
synapse/lib/lmdbslab.py
CHANGED
|
@@ -1623,7 +1623,17 @@ class Slab(s_base.Base):
|
|
|
1623
1623
|
self.commitstats.append((starttime, xactopslen, delta))
|
|
1624
1624
|
|
|
1625
1625
|
if self.WARN_COMMIT_TIME_MS and delta > self.WARN_COMMIT_TIME_MS:
|
|
1626
|
-
|
|
1626
|
+
|
|
1627
|
+
extra = {
|
|
1628
|
+
'delta': delta,
|
|
1629
|
+
'path': self.path,
|
|
1630
|
+
'sysctls': s_thisplat.getSysctls(),
|
|
1631
|
+
'xactopslen': xactopslen,
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
mesg = f'Commit with {xactopslen} items in {self!r} took {delta} ms - performance may be degraded.'
|
|
1635
|
+
logger.warning(mesg, extra={'synapse': extra})
|
|
1636
|
+
|
|
1627
1637
|
self._initCoXact()
|
|
1628
1638
|
return True
|
|
1629
1639
|
|
synapse/lib/modelrev.py
CHANGED
|
@@ -8,7 +8,7 @@ import synapse.lib.layer as s_layer
|
|
|
8
8
|
|
|
9
9
|
logger = logging.getLogger(__name__)
|
|
10
10
|
|
|
11
|
-
maxvers = (0, 2,
|
|
11
|
+
maxvers = (0, 2, 25)
|
|
12
12
|
|
|
13
13
|
class ModelRev:
|
|
14
14
|
|
|
@@ -38,6 +38,7 @@ class ModelRev:
|
|
|
38
38
|
((0, 2, 22), self.revModel_0_2_22),
|
|
39
39
|
((0, 2, 23), self.revModel_0_2_23),
|
|
40
40
|
((0, 2, 24), self.revModel_0_2_24),
|
|
41
|
+
((0, 2, 25), self.revModel_0_2_25),
|
|
41
42
|
)
|
|
42
43
|
|
|
43
44
|
async def _uniqSortArray(self, todoprops, layers):
|
|
@@ -757,6 +758,17 @@ class ModelRev:
|
|
|
757
758
|
for form, props in formprops.items():
|
|
758
759
|
await self._normVelocityProps(layers, form, props)
|
|
759
760
|
|
|
761
|
+
async def revModel_0_2_25(self, layers):
|
|
762
|
+
await self._typeToForm(layers, 'econ:currency', 'econ:currency')
|
|
763
|
+
await self._normPropValu(layers, 'ou:position:title')
|
|
764
|
+
await self._propToForm(layers, 'ou:position:title', 'ou:jobtitle')
|
|
765
|
+
|
|
766
|
+
await self._normPropValu(layers, 'ou:conference:name')
|
|
767
|
+
await self._propToForm(layers, 'ou:conference:name', 'entity:name')
|
|
768
|
+
|
|
769
|
+
await self._normPropValu(layers, 'ou:conference:names')
|
|
770
|
+
await self._propArrayToForm(layers, 'ou:conference:names', 'entity:name')
|
|
771
|
+
|
|
760
772
|
async def runStorm(self, text, opts=None):
|
|
761
773
|
'''
|
|
762
774
|
Run storm code in a schedcoro and log the output messages.
|
|
@@ -1116,6 +1128,10 @@ class ModelRev:
|
|
|
1116
1128
|
'''
|
|
1117
1129
|
await self.runStorm(storm, opts=opts)
|
|
1118
1130
|
|
|
1131
|
+
async def _typeToForm(self, layers, typename, formname):
|
|
1132
|
+
for prop in layers[0].core.model.getPropsByType(typename):
|
|
1133
|
+
await self._propToForm(layers, prop.full, formname)
|
|
1134
|
+
|
|
1119
1135
|
async def _propArrayToForm(self, layers, propfull, formname):
|
|
1120
1136
|
|
|
1121
1137
|
opts = {'vars': {
|
synapse/lib/modules.py
CHANGED
|
@@ -17,6 +17,7 @@ coremods = (
|
|
|
17
17
|
'synapse.models.telco.TelcoModule',
|
|
18
18
|
'synapse.models.inet.InetModule',
|
|
19
19
|
'synapse.models.material.MatModule',
|
|
20
|
+
'synapse.models.entity.EntityModule',
|
|
20
21
|
'synapse.models.language.LangModule',
|
|
21
22
|
'synapse.models.crypto.CryptoModule',
|
|
22
23
|
'synapse.models.gov.cn.GovCnModule',
|
synapse/lib/msgpack.py
CHANGED
|
@@ -7,8 +7,27 @@ import synapse.exc as s_exc
|
|
|
7
7
|
|
|
8
8
|
logger = logging.getLogger(__name__)
|
|
9
9
|
|
|
10
|
+
def _ext_un(code, byts):
|
|
11
|
+
if code == 0:
|
|
12
|
+
return int.from_bytes(byts, 'big')
|
|
13
|
+
elif code == 1:
|
|
14
|
+
return int.from_bytes(byts, 'big', signed=True)
|
|
15
|
+
else: # pragma: no cover
|
|
16
|
+
mesg = f'Invalid msgpack ext code: {code} ({repr(byts)[:20]})'
|
|
17
|
+
raise s_exc.SynErr(mesg=mesg)
|
|
18
|
+
|
|
19
|
+
def _ext_en(item):
|
|
20
|
+
if isinstance(item, int):
|
|
21
|
+
if item > 0xffffffffffffffff:
|
|
22
|
+
size = (item.bit_length() + 7) // 8
|
|
23
|
+
return msgpack.ExtType(0, item.to_bytes(size, 'big'))
|
|
24
|
+
if item < -0x8000000000000000:
|
|
25
|
+
size = (item.bit_length() // 8) + 1
|
|
26
|
+
return msgpack.ExtType(1, item.to_bytes(size, 'big', signed=True))
|
|
27
|
+
return item
|
|
28
|
+
|
|
10
29
|
# Single Packer object which is reused for performance
|
|
11
|
-
pakr = msgpack.Packer(use_bin_type=True, unicode_errors='surrogatepass')
|
|
30
|
+
pakr = msgpack.Packer(use_bin_type=True, unicode_errors='surrogatepass', default=_ext_en)
|
|
12
31
|
if isinstance(pakr, m_fallback.Packer): # pragma: no cover
|
|
13
32
|
logger.warning('******************************************************************************************************')
|
|
14
33
|
logger.warning('* msgpack is using the pure python fallback implementation. This will impact performance negatively. *')
|
|
@@ -21,6 +40,7 @@ unpacker_kwargs = {
|
|
|
21
40
|
'raw': False,
|
|
22
41
|
'use_list': False,
|
|
23
42
|
'strict_map_key': False,
|
|
43
|
+
'ext_hook': _ext_un,
|
|
24
44
|
'max_buffer_size': 2**32 - 1,
|
|
25
45
|
'unicode_errors': 'surrogatepass'
|
|
26
46
|
}
|
|
@@ -67,7 +87,8 @@ def _fallback_en(item):
|
|
|
67
87
|
bytes: The serialized bytes in msgpack format.
|
|
68
88
|
'''
|
|
69
89
|
try:
|
|
70
|
-
return msgpack.packb(item, use_bin_type=True,
|
|
90
|
+
return msgpack.packb(item, use_bin_type=True,
|
|
91
|
+
unicode_errors='surrogatepass', default=_ext_en)
|
|
71
92
|
except TypeError as e:
|
|
72
93
|
mesg = f'{e.args[0]}: {repr(item)[:20]}'
|
|
73
94
|
raise s_exc.NotMsgpackSafe(mesg=mesg) from e
|
|
@@ -95,7 +116,8 @@ def un(byts, use_list=False):
|
|
|
95
116
|
obj: The de-serialized object
|
|
96
117
|
'''
|
|
97
118
|
# This uses a subset of unpacker_kwargs
|
|
98
|
-
return msgpack.loads(byts, use_list=use_list, raw=False, strict_map_key=False,
|
|
119
|
+
return msgpack.loads(byts, use_list=use_list, raw=False, strict_map_key=False,
|
|
120
|
+
unicode_errors='surrogatepass', ext_hook=_ext_un)
|
|
99
121
|
|
|
100
122
|
def isok(item):
|
|
101
123
|
'''
|
synapse/lib/nexus.py
CHANGED
|
@@ -12,7 +12,6 @@ import synapse.common as s_common
|
|
|
12
12
|
import synapse.telepath as s_telepath
|
|
13
13
|
|
|
14
14
|
import synapse.lib.base as s_base
|
|
15
|
-
import synapse.lib.version as s_version
|
|
16
15
|
|
|
17
16
|
logger = logging.getLogger(__name__)
|
|
18
17
|
|
|
@@ -267,7 +266,7 @@ class NexsRoot(s_base.Base):
|
|
|
267
266
|
raise
|
|
268
267
|
|
|
269
268
|
except Exception:
|
|
270
|
-
logger.exception('Exception while replaying log')
|
|
269
|
+
logger.exception(f'Exception while replaying log: {s_common.trimText(repr(indxitem))}')
|
|
271
270
|
|
|
272
271
|
async def addWriteHold(self, reason):
|
|
273
272
|
|
|
@@ -497,27 +496,32 @@ class NexsRoot(s_base.Base):
|
|
|
497
496
|
|
|
498
497
|
async def runMirrorLoop(self, proxy):
|
|
499
498
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
499
|
+
try:
|
|
500
|
+
cellinfo = await proxy.getCellInfo()
|
|
501
|
+
features = cellinfo.get('features', {})
|
|
502
|
+
if features.get('dynmirror'):
|
|
503
|
+
await proxy.readyToMirror()
|
|
504
|
+
|
|
505
|
+
cellvers = cellinfo['cell']['version']
|
|
506
|
+
if cellvers > self.cell.VERSION:
|
|
507
|
+
logger.error('Leader is a higher version than we are. Mirrors must be updated first. Entering read-only mode.')
|
|
508
|
+
await self.addWriteHold(leaderversion)
|
|
509
|
+
# this will fire again on reconnect...
|
|
510
|
+
return
|
|
504
511
|
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
logger.error('Leader is a higher version than we are. Mirrors must be updated first. Entering read-only mode.')
|
|
508
|
-
await self.addWriteHold(leaderversion)
|
|
509
|
-
# this will fire again on reconnect...
|
|
510
|
-
return
|
|
512
|
+
# When we reconnect and the leader version has become ok...
|
|
513
|
+
await self.delWriteHold(leaderversion)
|
|
511
514
|
|
|
512
|
-
|
|
513
|
-
|
|
515
|
+
if self.celliden is not None:
|
|
516
|
+
if self.celliden != await proxy.getCellIden():
|
|
517
|
+
logger.error('remote cell has different iden! Aborting mirror sync')
|
|
518
|
+
await proxy.fini()
|
|
519
|
+
await self.fini()
|
|
520
|
+
return
|
|
514
521
|
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
await proxy.fini()
|
|
519
|
-
await self.fini()
|
|
520
|
-
return
|
|
522
|
+
except Exception as exc:
|
|
523
|
+
logger.exception(f'Unknown error during mirror loop startup: {exc}')
|
|
524
|
+
await proxy.fini()
|
|
521
525
|
|
|
522
526
|
while not proxy.isfini:
|
|
523
527
|
|
|
@@ -573,8 +577,8 @@ class NexsRoot(s_base.Base):
|
|
|
573
577
|
if respfutu is not None:
|
|
574
578
|
respfutu.set_result(retn)
|
|
575
579
|
|
|
576
|
-
except Exception: # pragma: no cover
|
|
577
|
-
logger.exception('error in mirror loop')
|
|
580
|
+
except Exception as exc: # pragma: no cover
|
|
581
|
+
logger.exception(f'error in mirror loop: {exc}')
|
|
578
582
|
|
|
579
583
|
# If we've left the mirror loop for some reason, we no longer know if we
|
|
580
584
|
# will be in the realtime window or not. So we should try to set the ready
|
synapse/lib/schemas.py
CHANGED
|
@@ -287,3 +287,34 @@ _stormPoolOptsSchema = {
|
|
|
287
287
|
'additionalProperties': False,
|
|
288
288
|
}
|
|
289
289
|
reqValidStormPoolOpts = s_config.getJsValidator(_stormPoolOptsSchema)
|
|
290
|
+
|
|
291
|
+
# These types are order sensitive
|
|
292
|
+
_changelogTypes = {'migration': 'Automatic Migrations',
|
|
293
|
+
'model': 'Model Changes',
|
|
294
|
+
'feat': 'Features and Enhancements',
|
|
295
|
+
'bug': 'Bugfixes',
|
|
296
|
+
'doc': 'Improved documentation',
|
|
297
|
+
'deprecation': 'Deprecations'}
|
|
298
|
+
|
|
299
|
+
_changelogSchema = {
|
|
300
|
+
'type': 'object',
|
|
301
|
+
'properties': {
|
|
302
|
+
'type': {
|
|
303
|
+
'type': 'string',
|
|
304
|
+
'enum': list(_changelogTypes.keys()),
|
|
305
|
+
},
|
|
306
|
+
'desc': {
|
|
307
|
+
'type': 'string',
|
|
308
|
+
'minLength': 1,
|
|
309
|
+
},
|
|
310
|
+
'prs': {
|
|
311
|
+
'type': 'array',
|
|
312
|
+
'items': {
|
|
313
|
+
'type': 'integer',
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
},
|
|
317
|
+
'additionalProperties': False,
|
|
318
|
+
'required': ['type', 'desc']
|
|
319
|
+
}
|
|
320
|
+
_reqChanglogSchema = s_config.getJsValidator(_changelogSchema)
|
synapse/lib/stormsvc.py
CHANGED
|
@@ -4,6 +4,7 @@ import logging
|
|
|
4
4
|
import synapse.telepath as s_telepath
|
|
5
5
|
|
|
6
6
|
import synapse.lib.base as s_base
|
|
7
|
+
import synapse.lib.hashitem as s_hashitem
|
|
7
8
|
|
|
8
9
|
logger = logging.getLogger(__name__)
|
|
9
10
|
|
|
@@ -139,32 +140,50 @@ class StormSvcClient(s_base.Base):
|
|
|
139
140
|
self.core.svcsbysvcname[self.svcname] = self
|
|
140
141
|
|
|
141
142
|
await self.core.feedBeholder('svc:set', {'name': self.name, 'iden': self.iden, 'svcname': self.svcname, 'version': self.svcvers})
|
|
143
|
+
# if the old service is the same as the new service, just skip
|
|
142
144
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
except Exception:
|
|
150
|
-
logger.exception(f'_delStormSvcPkgs failed for service {self.name} ({self.iden})')
|
|
145
|
+
oldpkgs = self.core.getStormSvcPkgs(self.iden)
|
|
146
|
+
byname = {}
|
|
147
|
+
done = set()
|
|
148
|
+
for pdef in oldpkgs:
|
|
149
|
+
iden = s_hashitem.hashitem(pdef)
|
|
150
|
+
byname[pdef.get('name')] = iden
|
|
151
151
|
|
|
152
152
|
# Register new packages
|
|
153
153
|
for pdef in self.info.get('pkgs', ()):
|
|
154
|
-
|
|
155
154
|
try:
|
|
156
|
-
# push the svciden in the package metadata for later reference.
|
|
157
155
|
pdef['svciden'] = self.iden
|
|
158
156
|
await self.core._normStormPkg(pdef)
|
|
157
|
+
except Exception:
|
|
158
|
+
name = pdef.get('name')
|
|
159
|
+
logger.exception(f'normStormPkg ({name}) failed for service {self.name} ({self.iden})')
|
|
160
|
+
continue
|
|
161
|
+
|
|
162
|
+
name = pdef.get('name')
|
|
163
|
+
iden = s_hashitem.hashitem(pdef)
|
|
164
|
+
|
|
165
|
+
done.add(name)
|
|
166
|
+
if name in byname:
|
|
167
|
+
if byname[name] != iden:
|
|
168
|
+
await self.core._delStormPkg(name) # we're updating an old package, so delete the old and then re-add
|
|
169
|
+
else:
|
|
170
|
+
continue # pkg unchanged. Can skip.
|
|
171
|
+
|
|
172
|
+
try:
|
|
173
|
+
# push the svciden in the package metadata for later reference.
|
|
159
174
|
await self.core._addStormPkg(pdef)
|
|
160
175
|
|
|
161
176
|
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
|
|
162
177
|
raise
|
|
163
178
|
|
|
164
179
|
except Exception:
|
|
165
|
-
name = pdef.get('name')
|
|
166
180
|
logger.exception(f'addStormPkg ({name}) failed for service {self.name} ({self.iden})')
|
|
167
181
|
|
|
182
|
+
# clean up any packages that no longer exist
|
|
183
|
+
for name in byname.keys():
|
|
184
|
+
if name not in done:
|
|
185
|
+
await self.core._delStormPkg(name)
|
|
186
|
+
|
|
168
187
|
# Set events and fire as needed
|
|
169
188
|
evts = self.info.get('evts')
|
|
170
189
|
try:
|
synapse/lib/stormtypes.py
CHANGED
|
@@ -18,7 +18,6 @@ import calendar
|
|
|
18
18
|
import functools
|
|
19
19
|
import contextlib
|
|
20
20
|
import collections
|
|
21
|
-
from typing import Any
|
|
22
21
|
|
|
23
22
|
import synapse.exc as s_exc
|
|
24
23
|
import synapse.common as s_common
|
|
@@ -8440,7 +8439,9 @@ class Trigger(Prim):
|
|
|
8440
8439
|
|
|
8441
8440
|
async def set(self, name, valu):
|
|
8442
8441
|
trigiden = self.valu.get('iden')
|
|
8443
|
-
viewiden = self.
|
|
8442
|
+
viewiden = self.valu.get('view')
|
|
8443
|
+
|
|
8444
|
+
view = self.runt.snap.core.reqView(viewiden)
|
|
8444
8445
|
|
|
8445
8446
|
name = await tostr(name)
|
|
8446
8447
|
if name in ('async', 'enabled', ):
|
|
@@ -8449,11 +8450,11 @@ class Trigger(Prim):
|
|
|
8449
8450
|
valu = await tostr(valu)
|
|
8450
8451
|
|
|
8451
8452
|
if name == 'user':
|
|
8452
|
-
self.runt.
|
|
8453
|
+
self.runt.confirm(('trigger', 'set', 'user'))
|
|
8453
8454
|
else:
|
|
8454
|
-
self.runt.
|
|
8455
|
+
self.runt.confirm(('trigger', 'set', name), gateiden=viewiden)
|
|
8455
8456
|
|
|
8456
|
-
await
|
|
8457
|
+
await view.setTriggerInfo(trigiden, name, valu)
|
|
8457
8458
|
|
|
8458
8459
|
self.valu[name] = valu
|
|
8459
8460
|
|
|
@@ -8825,6 +8826,8 @@ class LibCron(Lib):
|
|
|
8825
8826
|
'desc': 'Permits a user to create a cron job.'},
|
|
8826
8827
|
{'perm': ('cron', 'del'), 'gate': 'cronjob',
|
|
8827
8828
|
'desc': 'Permits a user to remove a cron job.'},
|
|
8829
|
+
{'perm': ('cron', 'kill'), 'gate': 'cronjob',
|
|
8830
|
+
'desc': 'Controls the ability to terminate a running cron job.'},
|
|
8828
8831
|
{'perm': ('cron', 'get'), 'gate': 'cronjob',
|
|
8829
8832
|
'desc': 'Permits a user to list cron jobs.'},
|
|
8830
8833
|
{'perm': ('cron', 'set'), 'gate': 'cronjob',
|
|
@@ -9276,14 +9279,19 @@ class CronJob(Prim):
|
|
|
9276
9279
|
{'name': 'valu', 'type': 'any', 'desc': 'The value to set on the definition.', },
|
|
9277
9280
|
),
|
|
9278
9281
|
'returns': {'type': 'cronjob', 'desc': 'The ``cronjob``', }}},
|
|
9282
|
+
|
|
9283
|
+
{'name': 'kill', 'desc': 'If the job is currently running, terminate the task.',
|
|
9284
|
+
'type': {'type': 'function', '_funcname': '_methCronJobKill',
|
|
9285
|
+
'returns': {'type': 'boolean', 'desc': 'A boolean value which is true if the task was terminated.'}}},
|
|
9286
|
+
|
|
9279
9287
|
{'name': 'pack', 'desc': 'Get the Cronjob definition.',
|
|
9280
9288
|
'type': {'type': 'function', '_funcname': '_methCronJobPack',
|
|
9281
|
-
'returns': {'type': 'dict', 'desc': 'The definition.'
|
|
9289
|
+
'returns': {'type': 'dict', 'desc': 'The definition.'}}},
|
|
9282
9290
|
{'name': 'pprint', 'desc': 'Get a dictionary containing user friendly strings for printing the CronJob.',
|
|
9283
9291
|
'type': {'type': 'function', '_funcname': '_methCronJobPprint',
|
|
9284
9292
|
'returns':
|
|
9285
9293
|
{'type': 'dict',
|
|
9286
|
-
'desc': 'A dictionary containing structured data about a cronjob for display purposes.'
|
|
9294
|
+
'desc': 'A dictionary containing structured data about a cronjob for display purposes.'}}},
|
|
9287
9295
|
)
|
|
9288
9296
|
_storm_typename = 'cronjob'
|
|
9289
9297
|
_ismutable = False
|
|
@@ -9300,10 +9308,16 @@ class CronJob(Prim):
|
|
|
9300
9308
|
def getObjLocals(self):
|
|
9301
9309
|
return {
|
|
9302
9310
|
'set': self._methCronJobSet,
|
|
9311
|
+
'kill': self._methCronJobKill,
|
|
9303
9312
|
'pack': self._methCronJobPack,
|
|
9304
9313
|
'pprint': self._methCronJobPprint,
|
|
9305
9314
|
}
|
|
9306
9315
|
|
|
9316
|
+
async def _methCronJobKill(self):
|
|
9317
|
+
iden = self.valu.get('iden')
|
|
9318
|
+
self.runt.confirm(('cron', 'kill'), gateiden=iden)
|
|
9319
|
+
return await self.runt.snap.core.killCronTask(iden)
|
|
9320
|
+
|
|
9307
9321
|
async def _methCronJobSet(self, name, valu):
|
|
9308
9322
|
name = await tostr(name)
|
|
9309
9323
|
valu = await toprim(valu)
|
|
@@ -9312,9 +9326,9 @@ class CronJob(Prim):
|
|
|
9312
9326
|
if name == 'creator':
|
|
9313
9327
|
# this permission must be granted cortex wide
|
|
9314
9328
|
# to prevent abuse...
|
|
9315
|
-
self.runt.
|
|
9329
|
+
self.runt.confirm(('cron', 'set', 'creator'))
|
|
9316
9330
|
else:
|
|
9317
|
-
self.runt.
|
|
9331
|
+
self.runt.confirm(('cron', 'set', name), gateiden=iden)
|
|
9318
9332
|
|
|
9319
9333
|
self.valu = await self.runt.snap.core.editCronJob(iden, name, valu)
|
|
9320
9334
|
|
synapse/lib/trigger.py
CHANGED
|
@@ -522,10 +522,6 @@ class Trigger:
|
|
|
522
522
|
logger.warning(f'Skipping trigger execution because user {self.user.iden} is locked')
|
|
523
523
|
return
|
|
524
524
|
|
|
525
|
-
tag = self.tdef.get('tag')
|
|
526
|
-
cond = self.tdef.get('cond')
|
|
527
|
-
form = self.tdef.get('form')
|
|
528
|
-
prop = self.tdef.get('prop')
|
|
529
525
|
storm = self.tdef.get('storm')
|
|
530
526
|
|
|
531
527
|
query = await self.view.core.getStormQuery(storm)
|
synapse/lib/version.py
CHANGED
|
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
|
|
|
223
223
|
##############################################################################
|
|
224
224
|
# The following are touched during the release process by bumpversion.
|
|
225
225
|
# Do not modify these directly.
|
|
226
|
-
version = (2,
|
|
226
|
+
version = (2, 174, 0)
|
|
227
227
|
verstring = '.'.join([str(x) for x in version])
|
|
228
|
-
commit = '
|
|
228
|
+
commit = 'cc04ec913185e0c1ef5a94958d6eb40eb288f38f'
|
synapse/lib/view.py
CHANGED
|
@@ -1518,6 +1518,8 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1518
1518
|
elif self.triggers.get(iden) is not None:
|
|
1519
1519
|
raise s_exc.DupIden(mesg='A trigger with this iden already exists')
|
|
1520
1520
|
|
|
1521
|
+
tdef['view'] = self.iden
|
|
1522
|
+
|
|
1521
1523
|
root = await self.core.auth.getUserByName('root')
|
|
1522
1524
|
|
|
1523
1525
|
tdef.setdefault('created', s_common.now())
|