synapse 2.193.0__py311-none-any.whl → 2.194.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/cortex.py +3 -7
- synapse/datamodel.py +6 -3
- synapse/exc.py +1 -1
- synapse/lib/agenda.py +17 -4
- synapse/lib/ast.py +217 -86
- synapse/lib/auth.py +1 -0
- synapse/lib/parser.py +4 -0
- synapse/lib/snap.py +40 -11
- synapse/lib/storm.lark +16 -1
- synapse/lib/storm.py +6 -4
- synapse/lib/storm_format.py +1 -0
- synapse/lib/stormctrl.py +88 -6
- synapse/lib/stormlib/cache.py +6 -2
- synapse/lib/stormlib/scrape.py +1 -1
- synapse/lib/stormlib/stix.py +8 -8
- synapse/lib/stormtypes.py +13 -5
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +20 -3
- synapse/models/geopol.py +1 -0
- synapse/models/geospace.py +1 -0
- synapse/models/inet.py +3 -0
- synapse/models/infotech.py +10 -2
- synapse/models/orgs.py +7 -2
- synapse/models/person.py +15 -4
- synapse/models/risk.py +3 -0
- synapse/models/telco.py +10 -3
- synapse/tests/test_axon.py +6 -6
- synapse/tests/test_cortex.py +130 -11
- synapse/tests/test_exc.py +1 -0
- synapse/tests/test_lib_agenda.py +125 -1
- synapse/tests/test_lib_aha.py +13 -6
- synapse/tests/test_lib_ast.py +258 -9
- synapse/tests/test_lib_auth.py +6 -7
- synapse/tests/test_lib_grammar.py +14 -0
- synapse/tests/test_lib_layer.py +1 -1
- synapse/tests/test_lib_lmdbslab.py +3 -3
- synapse/tests/test_lib_storm.py +201 -25
- synapse/tests/test_lib_stormctrl.py +65 -0
- synapse/tests/test_lib_stormhttp.py +5 -5
- synapse/tests/test_lib_stormlib_auth.py +5 -5
- synapse/tests/test_lib_stormlib_cache.py +38 -6
- synapse/tests/test_lib_stormlib_modelext.py +3 -3
- synapse/tests/test_lib_stormlib_scrape.py +4 -4
- synapse/tests/test_lib_stormlib_spooled.py +1 -1
- synapse/tests/test_lib_stormlib_xml.py +5 -5
- synapse/tests/test_lib_stormtypes.py +54 -57
- synapse/tests/test_lib_view.py +1 -1
- synapse/tests/test_model_base.py +1 -2
- synapse/tests/test_model_geopol.py +4 -0
- synapse/tests/test_model_geospace.py +6 -0
- synapse/tests/test_model_inet.py +3 -0
- synapse/tests/test_model_infotech.py +10 -1
- synapse/tests/test_model_orgs.py +17 -2
- synapse/tests/test_model_person.py +23 -1
- synapse/tests/test_model_risk.py +11 -0
- synapse/tests/test_tools_healthcheck.py +4 -4
- synapse/tests/test_utils.py +17 -18
- synapse/tests/utils.py +0 -35
- synapse/tools/changelog.py +6 -4
- synapse/tools/storm.py +1 -1
- {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/METADATA +5 -5
- {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/RECORD +65 -64
- {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/WHEEL +1 -1
- {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/LICENSE +0 -0
- {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py
CHANGED
|
@@ -1556,7 +1556,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
1556
1556
|
async def initServiceActive(self):
|
|
1557
1557
|
|
|
1558
1558
|
await self.stormdmons.start()
|
|
1559
|
-
await self.agenda.clearRunningStatus()
|
|
1560
1559
|
|
|
1561
1560
|
async def _runMigrations():
|
|
1562
1561
|
# Run migrations when this cortex becomes active. This is to prevent
|
|
@@ -5910,7 +5909,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5910
5909
|
opts = self._initStormOpts(opts)
|
|
5911
5910
|
|
|
5912
5911
|
if self.stormpool is not None and opts.get('mirror', True):
|
|
5913
|
-
extra = await self.getLogExtra(text=text)
|
|
5914
5912
|
proxy = await self._getMirrorProxy(opts)
|
|
5915
5913
|
|
|
5916
5914
|
if proxy is not None:
|
|
@@ -5930,7 +5928,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5930
5928
|
|
|
5931
5929
|
except s_exc.TimeOut:
|
|
5932
5930
|
mesg = 'Timeout waiting for query mirror, running locally instead.'
|
|
5933
|
-
logger.warning(mesg)
|
|
5931
|
+
logger.warning(mesg, extra=extra)
|
|
5934
5932
|
|
|
5935
5933
|
if (nexsoffs := opts.get('nexsoffs')) is not None:
|
|
5936
5934
|
if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
|
|
@@ -5945,7 +5943,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5945
5943
|
opts = self._initStormOpts(opts)
|
|
5946
5944
|
|
|
5947
5945
|
if self.stormpool is not None and opts.get('mirror', True):
|
|
5948
|
-
extra = await self.getLogExtra(text=text)
|
|
5949
5946
|
proxy = await self._getMirrorProxy(opts)
|
|
5950
5947
|
|
|
5951
5948
|
if proxy is not None:
|
|
@@ -5962,7 +5959,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5962
5959
|
return await proxy.callStorm(text, opts=mirropts)
|
|
5963
5960
|
except s_exc.TimeOut:
|
|
5964
5961
|
mesg = 'Timeout waiting for query mirror, running locally instead.'
|
|
5965
|
-
logger.warning(mesg)
|
|
5962
|
+
logger.warning(mesg, extra=extra)
|
|
5966
5963
|
|
|
5967
5964
|
if (nexsoffs := opts.get('nexsoffs')) is not None:
|
|
5968
5965
|
if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
|
|
@@ -5975,7 +5972,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5975
5972
|
opts = self._initStormOpts(opts)
|
|
5976
5973
|
|
|
5977
5974
|
if self.stormpool is not None and opts.get('mirror', True):
|
|
5978
|
-
extra = await self.getLogExtra(text=text)
|
|
5979
5975
|
proxy = await self._getMirrorProxy(opts)
|
|
5980
5976
|
|
|
5981
5977
|
if proxy is not None:
|
|
@@ -5995,7 +5991,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
|
|
|
5995
5991
|
|
|
5996
5992
|
except s_exc.TimeOut:
|
|
5997
5993
|
mesg = 'Timeout waiting for query mirror, running locally instead.'
|
|
5998
|
-
logger.warning(mesg)
|
|
5994
|
+
logger.warning(mesg, extra=extra)
|
|
5999
5995
|
|
|
6000
5996
|
if (nexsoffs := opts.get('nexsoffs')) is not None:
|
|
6001
5997
|
if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
|
synapse/datamodel.py
CHANGED
|
@@ -445,14 +445,17 @@ class Form:
|
|
|
445
445
|
'''
|
|
446
446
|
return self.props.get(name)
|
|
447
447
|
|
|
448
|
-
def reqProp(self, name):
|
|
448
|
+
def reqProp(self, name, extra=None):
|
|
449
449
|
prop = self.props.get(name)
|
|
450
450
|
if prop is not None:
|
|
451
451
|
return prop
|
|
452
452
|
|
|
453
453
|
full = f'{self.name}:{name}'
|
|
454
|
-
|
|
455
|
-
|
|
454
|
+
exc = s_exc.NoSuchProp.init(full)
|
|
455
|
+
if extra is not None:
|
|
456
|
+
exc = extra(exc)
|
|
457
|
+
|
|
458
|
+
raise exc
|
|
456
459
|
|
|
457
460
|
def pack(self):
|
|
458
461
|
props = {p.name: p.pack() for p in self.props.values()}
|
synapse/exc.py
CHANGED
synapse/lib/agenda.py
CHANGED
|
@@ -679,6 +679,11 @@ class Agenda(s_base.Base):
|
|
|
679
679
|
mesg = f'No cron job with iden: {iden}'
|
|
680
680
|
raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
|
|
681
681
|
|
|
682
|
+
self._delete_appt_from_heap(appt)
|
|
683
|
+
del self.appts[iden]
|
|
684
|
+
self.apptdefs.delete(iden)
|
|
685
|
+
|
|
686
|
+
def _delete_appt_from_heap(self, appt):
|
|
682
687
|
try:
|
|
683
688
|
heappos = self.apptheap.index(appt)
|
|
684
689
|
except ValueError:
|
|
@@ -692,9 +697,6 @@ class Agenda(s_base.Base):
|
|
|
692
697
|
self.apptheap[heappos] = self.apptheap.pop()
|
|
693
698
|
heapq.heapify(self.apptheap)
|
|
694
699
|
|
|
695
|
-
del self.appts[iden]
|
|
696
|
-
self.apptdefs.delete(iden)
|
|
697
|
-
|
|
698
700
|
def _getNowTick(self):
|
|
699
701
|
return time.time() + self.tickoff
|
|
700
702
|
|
|
@@ -707,12 +709,23 @@ class Agenda(s_base.Base):
|
|
|
707
709
|
for appt in list(self.appts.values()):
|
|
708
710
|
if appt.isrunning:
|
|
709
711
|
logger.debug(f'Clearing the isrunning flag for {appt.iden}')
|
|
710
|
-
|
|
712
|
+
|
|
713
|
+
edits = {
|
|
714
|
+
'isrunning': False,
|
|
715
|
+
'lastfinishtime': self._getNowTick(),
|
|
716
|
+
'lasterrs': ['aborted'] + appt.lasterrs[-4:]
|
|
717
|
+
}
|
|
718
|
+
await self.core.addCronEdits(appt.iden, edits)
|
|
719
|
+
await self.core.feedBeholder('cron:stop', {'iden': appt.iden})
|
|
720
|
+
|
|
721
|
+
if appt.nexttime is None:
|
|
722
|
+
self._delete_appt_from_heap(appt)
|
|
711
723
|
|
|
712
724
|
async def runloop(self):
|
|
713
725
|
'''
|
|
714
726
|
Task loop to issue query tasks at the right times.
|
|
715
727
|
'''
|
|
728
|
+
await self.clearRunningStatus()
|
|
716
729
|
while not self.isfini:
|
|
717
730
|
|
|
718
731
|
timeout = None
|