synapse 2.193.0__py311-none-any.whl → 2.194.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (65) hide show
  1. synapse/cortex.py +3 -7
  2. synapse/datamodel.py +6 -3
  3. synapse/exc.py +1 -1
  4. synapse/lib/agenda.py +17 -4
  5. synapse/lib/ast.py +217 -86
  6. synapse/lib/auth.py +1 -0
  7. synapse/lib/parser.py +4 -0
  8. synapse/lib/snap.py +40 -11
  9. synapse/lib/storm.lark +16 -1
  10. synapse/lib/storm.py +6 -4
  11. synapse/lib/storm_format.py +1 -0
  12. synapse/lib/stormctrl.py +88 -6
  13. synapse/lib/stormlib/cache.py +6 -2
  14. synapse/lib/stormlib/scrape.py +1 -1
  15. synapse/lib/stormlib/stix.py +8 -8
  16. synapse/lib/stormtypes.py +13 -5
  17. synapse/lib/version.py +2 -2
  18. synapse/lib/view.py +20 -3
  19. synapse/models/geopol.py +1 -0
  20. synapse/models/geospace.py +1 -0
  21. synapse/models/inet.py +3 -0
  22. synapse/models/infotech.py +10 -2
  23. synapse/models/orgs.py +7 -2
  24. synapse/models/person.py +15 -4
  25. synapse/models/risk.py +3 -0
  26. synapse/models/telco.py +10 -3
  27. synapse/tests/test_axon.py +6 -6
  28. synapse/tests/test_cortex.py +130 -11
  29. synapse/tests/test_exc.py +1 -0
  30. synapse/tests/test_lib_agenda.py +125 -1
  31. synapse/tests/test_lib_aha.py +13 -6
  32. synapse/tests/test_lib_ast.py +258 -9
  33. synapse/tests/test_lib_auth.py +6 -7
  34. synapse/tests/test_lib_grammar.py +14 -0
  35. synapse/tests/test_lib_layer.py +1 -1
  36. synapse/tests/test_lib_lmdbslab.py +3 -3
  37. synapse/tests/test_lib_storm.py +201 -25
  38. synapse/tests/test_lib_stormctrl.py +65 -0
  39. synapse/tests/test_lib_stormhttp.py +5 -5
  40. synapse/tests/test_lib_stormlib_auth.py +5 -5
  41. synapse/tests/test_lib_stormlib_cache.py +38 -6
  42. synapse/tests/test_lib_stormlib_modelext.py +3 -3
  43. synapse/tests/test_lib_stormlib_scrape.py +4 -4
  44. synapse/tests/test_lib_stormlib_spooled.py +1 -1
  45. synapse/tests/test_lib_stormlib_xml.py +5 -5
  46. synapse/tests/test_lib_stormtypes.py +54 -57
  47. synapse/tests/test_lib_view.py +1 -1
  48. synapse/tests/test_model_base.py +1 -2
  49. synapse/tests/test_model_geopol.py +4 -0
  50. synapse/tests/test_model_geospace.py +6 -0
  51. synapse/tests/test_model_inet.py +3 -0
  52. synapse/tests/test_model_infotech.py +10 -1
  53. synapse/tests/test_model_orgs.py +17 -2
  54. synapse/tests/test_model_person.py +23 -1
  55. synapse/tests/test_model_risk.py +11 -0
  56. synapse/tests/test_tools_healthcheck.py +4 -4
  57. synapse/tests/test_utils.py +17 -18
  58. synapse/tests/utils.py +0 -35
  59. synapse/tools/changelog.py +6 -4
  60. synapse/tools/storm.py +1 -1
  61. {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/METADATA +5 -5
  62. {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/RECORD +65 -64
  63. {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/WHEEL +1 -1
  64. {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/LICENSE +0 -0
  65. {synapse-2.193.0.dist-info → synapse-2.194.0.dist-info}/top_level.txt +0 -0
synapse/cortex.py CHANGED
@@ -1556,7 +1556,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
1556
1556
  async def initServiceActive(self):
1557
1557
 
1558
1558
  await self.stormdmons.start()
1559
- await self.agenda.clearRunningStatus()
1560
1559
 
1561
1560
  async def _runMigrations():
1562
1561
  # Run migrations when this cortex becomes active. This is to prevent
@@ -5910,7 +5909,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5910
5909
  opts = self._initStormOpts(opts)
5911
5910
 
5912
5911
  if self.stormpool is not None and opts.get('mirror', True):
5913
- extra = await self.getLogExtra(text=text)
5914
5912
  proxy = await self._getMirrorProxy(opts)
5915
5913
 
5916
5914
  if proxy is not None:
@@ -5930,7 +5928,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5930
5928
 
5931
5929
  except s_exc.TimeOut:
5932
5930
  mesg = 'Timeout waiting for query mirror, running locally instead.'
5933
- logger.warning(mesg)
5931
+ logger.warning(mesg, extra=extra)
5934
5932
 
5935
5933
  if (nexsoffs := opts.get('nexsoffs')) is not None:
5936
5934
  if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
@@ -5945,7 +5943,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5945
5943
  opts = self._initStormOpts(opts)
5946
5944
 
5947
5945
  if self.stormpool is not None and opts.get('mirror', True):
5948
- extra = await self.getLogExtra(text=text)
5949
5946
  proxy = await self._getMirrorProxy(opts)
5950
5947
 
5951
5948
  if proxy is not None:
@@ -5962,7 +5959,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5962
5959
  return await proxy.callStorm(text, opts=mirropts)
5963
5960
  except s_exc.TimeOut:
5964
5961
  mesg = 'Timeout waiting for query mirror, running locally instead.'
5965
- logger.warning(mesg)
5962
+ logger.warning(mesg, extra=extra)
5966
5963
 
5967
5964
  if (nexsoffs := opts.get('nexsoffs')) is not None:
5968
5965
  if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
@@ -5975,7 +5972,6 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5975
5972
  opts = self._initStormOpts(opts)
5976
5973
 
5977
5974
  if self.stormpool is not None and opts.get('mirror', True):
5978
- extra = await self.getLogExtra(text=text)
5979
5975
  proxy = await self._getMirrorProxy(opts)
5980
5976
 
5981
5977
  if proxy is not None:
@@ -5995,7 +5991,7 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore
5995
5991
 
5996
5992
  except s_exc.TimeOut:
5997
5993
  mesg = 'Timeout waiting for query mirror, running locally instead.'
5998
- logger.warning(mesg)
5994
+ logger.warning(mesg, extra=extra)
5999
5995
 
6000
5996
  if (nexsoffs := opts.get('nexsoffs')) is not None:
6001
5997
  if not await self.waitNexsOffs(nexsoffs, timeout=opts.get('nexstimeout')):
synapse/datamodel.py CHANGED
@@ -445,14 +445,17 @@ class Form:
445
445
  '''
446
446
  return self.props.get(name)
447
447
 
448
- def reqProp(self, name):
448
+ def reqProp(self, name, extra=None):
449
449
  prop = self.props.get(name)
450
450
  if prop is not None:
451
451
  return prop
452
452
 
453
453
  full = f'{self.name}:{name}'
454
- mesg = f'No property named {full}.'
455
- raise s_exc.NoSuchProp(mesg=mesg, name=full)
454
+ exc = s_exc.NoSuchProp.init(full)
455
+ if extra is not None:
456
+ exc = extra(exc)
457
+
458
+ raise exc
456
459
 
457
460
  def pack(self):
458
461
  props = {p.name: p.pack() for p in self.props.values()}
synapse/exc.py CHANGED
@@ -58,7 +58,7 @@ class SynErr(Exception):
58
58
 
59
59
  def update(self, items: dict):
60
60
  '''Update multiple items in the errinfo dict at once.'''
61
- self.errinfo.update(**items)
61
+ self.errinfo.update(items)
62
62
  self._setExcMesg()
63
63
 
64
64
  class StormRaise(SynErr):
synapse/lib/agenda.py CHANGED
@@ -679,6 +679,11 @@ class Agenda(s_base.Base):
679
679
  mesg = f'No cron job with iden: {iden}'
680
680
  raise s_exc.NoSuchIden(iden=iden, mesg=mesg)
681
681
 
682
+ self._delete_appt_from_heap(appt)
683
+ del self.appts[iden]
684
+ self.apptdefs.delete(iden)
685
+
686
+ def _delete_appt_from_heap(self, appt):
682
687
  try:
683
688
  heappos = self.apptheap.index(appt)
684
689
  except ValueError:
@@ -692,9 +697,6 @@ class Agenda(s_base.Base):
692
697
  self.apptheap[heappos] = self.apptheap.pop()
693
698
  heapq.heapify(self.apptheap)
694
699
 
695
- del self.appts[iden]
696
- self.apptdefs.delete(iden)
697
-
698
700
  def _getNowTick(self):
699
701
  return time.time() + self.tickoff
700
702
 
@@ -707,12 +709,23 @@ class Agenda(s_base.Base):
707
709
  for appt in list(self.appts.values()):
708
710
  if appt.isrunning:
709
711
  logger.debug(f'Clearing the isrunning flag for {appt.iden}')
710
- await self.core.addCronEdits(appt.iden, {'isrunning': False})
712
+
713
+ edits = {
714
+ 'isrunning': False,
715
+ 'lastfinishtime': self._getNowTick(),
716
+ 'lasterrs': ['aborted'] + appt.lasterrs[-4:]
717
+ }
718
+ await self.core.addCronEdits(appt.iden, edits)
719
+ await self.core.feedBeholder('cron:stop', {'iden': appt.iden})
720
+
721
+ if appt.nexttime is None:
722
+ self._delete_appt_from_heap(appt)
711
723
 
712
724
  async def runloop(self):
713
725
  '''
714
726
  Task loop to issue query tasks at the right times.
715
727
  '''
728
+ await self.clearRunningStatus()
716
729
  while not self.isfini:
717
730
 
718
731
  timeout = None