synapse 2.223.0__py311-none-any.whl → 2.225.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (135) hide show
  1. synapse/axon.py +10 -5
  2. synapse/common.py +2 -2
  3. synapse/cortex.py +52 -3
  4. synapse/datamodel.py +1 -1
  5. synapse/lib/cell.py +1 -1
  6. synapse/lib/const.py +4 -0
  7. synapse/lib/layer.py +6 -6
  8. synapse/lib/multislabseqn.py +36 -1
  9. synapse/lib/nexus.py +67 -8
  10. synapse/lib/queue.py +4 -1
  11. synapse/lib/rstorm.py +2 -2
  12. synapse/lib/schemas.py +13 -1
  13. synapse/lib/slabseqn.py +28 -0
  14. synapse/lib/storm.py +40 -2
  15. synapse/lib/stormhttp.py +7 -1
  16. synapse/lib/stormlib/imap.py +12 -4
  17. synapse/lib/stormlib/task.py +0 -1
  18. synapse/lib/stormtypes.py +19 -1
  19. synapse/lib/version.py +2 -2
  20. synapse/models/inet.py +29 -0
  21. synapse/models/media.py +4 -0
  22. synapse/models/proj.py +3 -0
  23. synapse/models/risk.py +9 -0
  24. synapse/models/syn.py +8 -0
  25. synapse/tests/test_common.py +4 -0
  26. synapse/tests/test_cortex.py +53 -2
  27. synapse/tests/test_lib_agenda.py +1 -1
  28. synapse/tests/test_lib_cell.py +1 -1
  29. synapse/tests/test_lib_certdir.py +1 -1
  30. synapse/tests/test_lib_httpapi.py +1 -1
  31. synapse/tests/test_lib_layer.py +1 -1
  32. synapse/tests/test_lib_multislabseqn.py +22 -0
  33. synapse/tests/test_lib_nexus.py +42 -1
  34. synapse/tests/test_lib_slabseqn.py +30 -1
  35. synapse/tests/test_lib_storm.py +156 -1
  36. synapse/tests/test_lib_stormhttp.py +16 -0
  37. synapse/tests/test_lib_stormlib_imap.py +14 -0
  38. synapse/tests/test_lib_stormlib_oauth.py +1 -1
  39. synapse/tests/test_lib_stormsvc.py +1 -1
  40. synapse/tests/test_lib_stormtypes.py +12 -0
  41. synapse/tests/test_lib_trigger.py +1 -1
  42. synapse/tests/test_model_inet.py +29 -0
  43. synapse/tests/test_model_media.py +4 -1
  44. synapse/tests/test_model_proj.py +3 -1
  45. synapse/tests/test_model_risk.py +12 -0
  46. synapse/tests/test_model_syn.py +54 -2
  47. synapse/tests/{test_tools_axon2axon.py → test_tools_axon_copy.py} +4 -4
  48. synapse/tests/{test_tools_pullfile.py → test_tools_axon_get.py} +4 -4
  49. synapse/tests/{test_tools_pushfile.py → test_tools_axon_put.py} +7 -7
  50. synapse/tests/{test_tools_csvtool.py → test_tools_cortex_csv.py} +12 -3
  51. synapse/tests/{test_tools_feed.py → test_tools_cortex_feed.py} +2 -2
  52. synapse/tests/{test_tools_apikey.py → test_tools_service_apikey.py} +1 -4
  53. synapse/tests/{test_tools_backup.py → test_tools_service_backup.py} +5 -5
  54. synapse/tests/{test_tools_demote.py → test_tools_service_demote.py} +1 -1
  55. synapse/tests/{test_tools_healthcheck.py → test_tools_service_healthcheck.py} +1 -1
  56. synapse/tests/{test_tools_livebackup.py → test_tools_service_livebackup.py} +1 -1
  57. synapse/tests/{test_tools_modrole.py → test_tools_service_modrole.py} +1 -1
  58. synapse/tests/{test_tools_moduser.py → test_tools_service_moduser.py} +1 -1
  59. synapse/tests/{test_tools_promote.py → test_tools_service_promote.py} +1 -1
  60. synapse/tests/{test_tools_reload.py → test_tools_service_reload.py} +1 -1
  61. synapse/tests/{test_tools_shutdown.py → test_tools_service_shutdown.py} +1 -1
  62. synapse/tests/{test_tools_snapshot.py → test_tools_service_snapshot.py} +1 -1
  63. synapse/tests/{test_tools_storm.py → test_tools_storm_cli.py} +1 -1
  64. synapse/tests/{test_tools_pkgs_gendocs.py → test_tools_storm_pkg_doc.py} +12 -3
  65. synapse/tests/{test_tools_genpkg.py → test_tools_storm_pkg_gen.py} +1 -1
  66. synapse/tests/{test_tools_autodoc.py → test_tools_utils_autodoc.py} +1 -1
  67. synapse/tests/test_tools_utils_changelog.py +454 -0
  68. synapse/tests/{test_tools_easycert.py → test_tools_utils_easycert.py} +48 -46
  69. synapse/tests/{test_tools_guid.py → test_tools_utils_guid.py} +3 -3
  70. synapse/tests/{test_tools_json2mpk.py → test_tools_utils_json2mpk.py} +3 -3
  71. synapse/tests/{test_tools_rstorm.py → test_tools_utils_rstorm.py} +6 -1
  72. synapse/tests/utils.py +3 -1
  73. synapse/tools/apikey.py +4 -83
  74. synapse/tools/autodoc.py +3 -1031
  75. synapse/tools/axon/copy.py +44 -0
  76. synapse/tools/axon/get.py +64 -0
  77. synapse/tools/axon/put.py +122 -0
  78. synapse/tools/axon2axon.py +3 -36
  79. synapse/tools/backup.py +6 -176
  80. synapse/tools/changelog.py +3 -1098
  81. synapse/tools/cortex/csv.py +236 -0
  82. synapse/tools/cortex/feed.py +151 -0
  83. synapse/tools/csvtool.py +3 -227
  84. synapse/tools/demote.py +4 -40
  85. synapse/tools/docker/validate.py +3 -3
  86. synapse/tools/easycert.py +4 -129
  87. synapse/tools/feed.py +3 -140
  88. synapse/tools/genpkg.py +3 -307
  89. synapse/tools/guid.py +7 -6
  90. synapse/tools/healthcheck.py +3 -101
  91. synapse/tools/json2mpk.py +6 -38
  92. synapse/tools/livebackup.py +4 -27
  93. synapse/tools/modrole.py +3 -108
  94. synapse/tools/moduser.py +3 -179
  95. synapse/tools/pkgs/gendocs.py +3 -164
  96. synapse/tools/promote.py +4 -41
  97. synapse/tools/pullfile.py +3 -56
  98. synapse/tools/pushfile.py +3 -114
  99. synapse/tools/reload.py +4 -61
  100. synapse/tools/rstorm.py +3 -26
  101. synapse/tools/service/__init__.py +0 -0
  102. synapse/tools/service/apikey.py +90 -0
  103. synapse/tools/service/backup.py +181 -0
  104. synapse/tools/service/demote.py +47 -0
  105. synapse/tools/service/healthcheck.py +109 -0
  106. synapse/tools/service/livebackup.py +34 -0
  107. synapse/tools/service/modrole.py +116 -0
  108. synapse/tools/service/moduser.py +184 -0
  109. synapse/tools/service/promote.py +48 -0
  110. synapse/tools/service/reload.py +68 -0
  111. synapse/tools/service/shutdown.py +51 -0
  112. synapse/tools/service/snapshot.py +64 -0
  113. synapse/tools/shutdown.py +5 -45
  114. synapse/tools/snapshot.py +4 -57
  115. synapse/tools/storm/__init__.py +0 -0
  116. synapse/tools/storm/__main__.py +5 -0
  117. synapse/tools/{storm.py → storm/_cli.py} +0 -3
  118. synapse/tools/storm/pkg/__init__.py +0 -0
  119. synapse/tools/{pkgs/pandoc_filter.py → storm/pkg/_pandoc_filter.py} +1 -1
  120. synapse/tools/storm/pkg/doc.py +176 -0
  121. synapse/tools/storm/pkg/gen.py +315 -0
  122. synapse/tools/utils/__init__.py +0 -0
  123. synapse/tools/utils/autodoc.py +1040 -0
  124. synapse/tools/utils/changelog.py +1124 -0
  125. synapse/tools/utils/easycert.py +136 -0
  126. synapse/tools/utils/guid.py +11 -0
  127. synapse/tools/utils/json2mpk.py +46 -0
  128. synapse/tools/utils/rstorm.py +35 -0
  129. {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/METADATA +1 -1
  130. {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/RECORD +134 -105
  131. synapse/tests/test_tools_changelog.py +0 -196
  132. /synapse/tests/{test_tools_axon.py → test_tools_axon_dump_load.py} +0 -0
  133. {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/WHEEL +0 -0
  134. {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/licenses/LICENSE +0 -0
  135. {synapse-2.223.0.dist-info → synapse-2.225.0.dist-info}/top_level.txt +0 -0
synapse/lib/storm.py CHANGED
@@ -932,6 +932,7 @@ stormcmds = (
932
932
  },
933
933
  {
934
934
  'name': 'ps.list',
935
+ 'deprecated': {'eolvers': 'v3.0.0', 'mesg': 'Use ``task.list`` instead.'},
935
936
  'descr': 'List running tasks in the cortex.',
936
937
  'cmdargs': (
937
938
  ('--verbose', {'default': False, 'action': 'store_true', 'help': 'Enable verbose output.'}),
@@ -958,6 +959,7 @@ stormcmds = (
958
959
  },
959
960
  {
960
961
  'name': 'ps.kill',
962
+ 'deprecated': {'eolvers': 'v3.0.0', 'mesg': 'Use ``task.kill`` instead.'},
961
963
  'descr': 'Kill a running task/query within the cortex.',
962
964
  'cmdargs': (
963
965
  ('iden', {'help': 'Any prefix that matches exactly one valid process iden is accepted.'}),
@@ -1111,6 +1113,16 @@ stormcmds = (
1111
1113
  },
1112
1114
  )
1113
1115
 
1116
+ def deprmesg(name, depritem):
1117
+ if (mesg := depritem.get('mesg')) is not None:
1118
+ return f'"{name}" is deprecated: {mesg}'
1119
+
1120
+ if (eolvers := depritem.get('eolvers')) is not None:
1121
+ return f'"{name}" is deprecated and will be removed in {eolvers}.'
1122
+
1123
+ eoldate = depritem.get('eoldate')
1124
+ return f'"{name}" is deprecated and will be removed on {eoldate}.'
1125
+
1114
1126
  @s_cache.memoize(size=1024)
1115
1127
  def queryhash(text):
1116
1128
  return s_common.queryhash(text)
@@ -1906,6 +1918,7 @@ class Parser:
1906
1918
  self.root = root
1907
1919
  self.exited = False
1908
1920
  self.mesgs = []
1921
+ self.deprs = {}
1909
1922
 
1910
1923
  self.optargs = {}
1911
1924
  self.posargs = []
@@ -1931,7 +1944,7 @@ class Parser:
1931
1944
 
1932
1945
  choices = opts.get('choices')
1933
1946
  if choices is not None and opts.get('action') in ('store_true', 'store_false'):
1934
- mesg = f'Argument choices are not supported when action is store_true or store_false'
1947
+ mesg = 'Argument choices are not supported when action is store_true or store_false'
1935
1948
  raise s_exc.BadArg(mesg=mesg, argtype=str(argtype))
1936
1949
 
1937
1950
  dest = self._get_dest(names)
@@ -1988,6 +2001,9 @@ class Parser:
1988
2001
 
1989
2002
  dest = argdef.get('dest')
1990
2003
 
2004
+ if (deprecated := argdef.get('deprecated')) is not None:
2005
+ self.deprs[item] = deprecated
2006
+
1991
2007
  oact = argdef.get('action', 'store')
1992
2008
  if oact == 'store_true':
1993
2009
  opts[dest] = True
@@ -2177,6 +2193,12 @@ class Parser:
2177
2193
 
2178
2194
  self._printf(f'Usage: {self.prog} [options] {posargs}')
2179
2195
 
2196
+ if self.cdef is not None and (deprecated := self.cdef.get('deprecated')) is not None:
2197
+ dmsg = deprmesg(self.prog, deprecated)
2198
+ self._printf('')
2199
+ self._printf(f'Deprecated: {dmsg}')
2200
+ self._printf('')
2201
+
2180
2202
  if self.cdef is not None and (endpoints := self.cdef.get('endpoints')):
2181
2203
  self._printf('')
2182
2204
  self._printf('Endpoints:')
@@ -2286,6 +2308,10 @@ class Parser:
2286
2308
  wrap_first = self._wrap_text(first, wrap_w)
2287
2309
  self._printf(f'{base:<{base_w-2}}: {wrap_first[0]}')
2288
2310
 
2311
+ if (deprecated := argdef.get('deprecated')) is not None:
2312
+ mesg = deprmesg(names[0], deprecated)
2313
+ self._printf(f'{"":<{base_w-2}} Deprecated: {mesg}')
2314
+
2289
2315
  for ln in wrap_first[1:]: self._printf(f'{"":<{base_w}}{ln}')
2290
2316
  for ln in helplst[1:]:
2291
2317
  lead_s = len(ln) - len(ln.lstrip())
@@ -2401,6 +2427,10 @@ class Cmd:
2401
2427
  except s_exc.BadSyntax: # pragma: no cover
2402
2428
  pass
2403
2429
 
2430
+ for item, depr in self.pars.deprs.items():
2431
+ mesg = deprmesg(item, depr)
2432
+ await self.runt.snap.warnonce(mesg)
2433
+
2404
2434
  for line in self.pars.mesgs:
2405
2435
  await self.runt.snap.printf(line)
2406
2436
 
@@ -2515,13 +2545,21 @@ class PureCmd(Cmd):
2515
2545
 
2516
2546
  cmdopts = s_stormtypes.CmdOpts(self)
2517
2547
 
2548
+ cmdconf = self.cdef.get('cmdconf', {})
2549
+ if cmdconf:
2550
+ cmdconf = s_msgpack.deepcopy(cmdconf, use_list=True)
2551
+
2518
2552
  opts = {
2519
2553
  'vars': {
2520
2554
  'cmdopts': cmdopts,
2521
- 'cmdconf': self.cdef.get('cmdconf', {}),
2555
+ 'cmdconf': cmdconf,
2522
2556
  }
2523
2557
  }
2524
2558
 
2559
+ if (deprecated := self.cdef.get('deprecated')) is not None:
2560
+ mesg = deprmesg(name, deprecated)
2561
+ await self.runt.warnonce(mesg)
2562
+
2525
2563
  if self.runtsafe:
2526
2564
  data = {'pathvars': {}}
2527
2565
  async def genx():
synapse/lib/stormhttp.py CHANGED
@@ -12,6 +12,7 @@ import synapse.common as s_common
12
12
 
13
13
  import synapse.lib.base as s_base
14
14
  import synapse.lib.json as s_json
15
+ import synapse.lib.const as s_const
15
16
  import synapse.lib.msgpack as s_msgpack
16
17
  import synapse.lib.version as s_version
17
18
  import synapse.lib.stormtypes as s_stormtypes
@@ -400,7 +401,12 @@ class LibHttp(s_stormtypes.Lib):
400
401
  proxy = await s_stormtypes.toprim(proxy)
401
402
  ssl_opts = await s_stormtypes.toprim(ssl_opts)
402
403
 
403
- kwargs = {'allow_redirects': allow_redirects}
404
+ kwargs = {
405
+ 'max_line_size': s_const.MAX_LINE_SIZE,
406
+ 'max_field_size': s_const.MAX_FIELD_SIZE,
407
+ 'allow_redirects': allow_redirects,
408
+ }
409
+
404
410
  if params:
405
411
  kwargs['params'] = s_stormtypes.strifyHttpArg(params, multi=True)
406
412
 
@@ -526,14 +526,22 @@ class ImapLib(s_stormtypes.Lib):
526
526
 
527
527
  try:
528
528
  imap = await s_common.wait_for(coro, timeout)
529
- except asyncio.TimeoutError:
529
+ except TimeoutError:
530
530
  raise s_exc.TimeOut(mesg='Timed out waiting for IMAP server hello.') from None
531
531
 
532
532
  async def fini():
533
- async def _logout():
534
- await s_common.wait_for(imap.logout(), 5)
533
+ async def imapfini():
534
+ if imap.isfini:
535
+ return
536
+
537
+ try:
538
+ await s_common.wait_for(imap.logout(), 5)
539
+ except TimeoutError:
540
+ pass # pragma: no cover
541
+
535
542
  await imap.fini()
536
- s_coro.create_task(_logout())
543
+
544
+ self.runt.snap.core.schedCoro(imapfini())
537
545
 
538
546
  self.runt.snap.onfini(fini)
539
547
 
@@ -1,4 +1,3 @@
1
- import asyncio
2
1
  import logging
3
2
 
4
3
  import synapse.exc as s_exc
synapse/lib/stormtypes.py CHANGED
@@ -1910,6 +1910,7 @@ class LibPs(Lib):
1910
1910
  'type': {'type': 'function', '_funcname': '_list',
1911
1911
  'returns': {'type': 'list', 'desc': 'A list of task definitions.', }}},
1912
1912
  )
1913
+ _storm_lib_deprecation = {'eolvers': 'v3.0.0', 'mesg': 'Use the corresponding ``$lib.task`` function.'}
1913
1914
  _storm_lib_path = ('ps',)
1914
1915
 
1915
1916
  def getObjLocals(self):
@@ -2968,7 +2969,13 @@ class LibTime(Lib):
2968
2969
 
2969
2970
  storm> $now=$lib.time.now() $str=$lib.time.format($now, '%A %d, %B %Y') $lib.print($str)
2970
2971
 
2971
- Tuesday 14, July 2020''',
2972
+ Tuesday 14, July 2020
2973
+
2974
+ Format a timestamp into a string using included format string::
2975
+
2976
+ storm> $now=$lib.time.now() $str=$lib.time.format($now, $lib.time.formats.iso8601) $lib.print($str)
2977
+
2978
+ 2025-10-02T09:34:00Z''',
2972
2979
  'type': {'type': 'function', '_funcname': '_format',
2973
2980
  'args': (
2974
2981
  {'name': 'valu', 'type': 'int', 'desc': 'A timestamp in epoch milliseconds.', },
@@ -3103,6 +3110,10 @@ class LibTime(Lib):
3103
3110
  {'name': 'timezone', 'desc': 'A timezone name. See python pytz docs for options.', 'type': 'str'},
3104
3111
  ),
3105
3112
  'returns': {'type': 'list', 'desc': 'An ($ok, $valu) tuple.', }}},
3113
+ {'name': 'formats.iso8601', 'desc': 'ISO8601 time format string in UTC timezone (Z).', 'type': 'str'},
3114
+ {'name': 'formats.iso8601us', 'desc': 'ISO8601 time format string (with microseconds) in UTC timezone (Z).', 'type': 'str'},
3115
+ {'name': 'formats.rfc2822', 'desc': 'RFC 2822 time format string in UT timezone.', 'type': 'str'},
3116
+ {'name': 'formats.synapse', 'desc': 'Synapse time format string.', 'type': 'str'},
3106
3117
  )
3107
3118
  _storm_lib_path = ('time',)
3108
3119
 
@@ -3127,6 +3138,13 @@ class LibTime(Lib):
3127
3138
  'dayofyear': self.dayofyear,
3128
3139
  'dayofmonth': self.dayofmonth,
3129
3140
  'monthofyear': self.monthofyear,
3141
+
3142
+ 'formats': {
3143
+ 'iso8601': '%Y-%m-%dT%H:%M:%SZ',
3144
+ 'iso8601us': '%Y-%m-%dT%H:%M:%S.%fZ',
3145
+ 'rfc2822': '%d %b %Y %H:%M:%S UT',
3146
+ 'synapse': '%Y/%m/%d %H:%M:%S.%f',
3147
+ },
3130
3148
  }
3131
3149
 
3132
3150
  @stormfunc(readonly=True)
synapse/lib/version.py CHANGED
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
223
223
  ##############################################################################
224
224
  # The following are touched during the release process by bumpversion.
225
225
  # Do not modify these directly.
226
- version = (2, 223, 0)
226
+ version = (2, 225, 0)
227
227
  verstring = '.'.join([str(x) for x in version])
228
- commit = 'b40e64b75fa2820856ad138b8ad9f8c4b6e9547e'
228
+ commit = 'd2ab923e3df9fe64611ac05b6ffdd7e8c2dc1b22'
synapse/models/inet.py CHANGED
@@ -3700,6 +3700,9 @@ class InetModule(s_module.CoreModule):
3700
3700
  )),
3701
3701
  ('inet:service:platform', {}, (
3702
3702
 
3703
+ ('id', ('str', {'strip': True}), {
3704
+ 'doc': 'An ID which identifies the platform.'}),
3705
+
3703
3706
  ('url', ('inet:url', {}), {
3704
3707
  'ex': 'https://twitter.com',
3705
3708
  'alts': ('urls',),
@@ -3708,6 +3711,13 @@ class InetModule(s_module.CoreModule):
3708
3711
  ('urls', ('array', {'type': 'inet:url', 'sorted': True, 'uniq': True}), {
3709
3712
  'doc': 'An array of alternate URLs for the platform.'}),
3710
3713
 
3714
+ ('zone', ('inet:fqdn', {}), {
3715
+ 'alts': ('zones',),
3716
+ 'doc': 'The primary zone for the platform.'}),
3717
+
3718
+ ('zones', ('array', {'type': 'inet:fqdn', 'sorted': True, 'uniq': True}), {
3719
+ 'doc': 'An array of alternate zones for the platform.'}),
3720
+
3711
3721
  ('name', ('str', {'onespace': True, 'lower': True}), {
3712
3722
  'ex': 'twitter',
3713
3723
  'alts': ('names',),
@@ -3722,6 +3732,21 @@ class InetModule(s_module.CoreModule):
3722
3732
  'disp': {'hint': 'text'},
3723
3733
  'doc': 'A description of the service platform.'}),
3724
3734
 
3735
+ ('parent', ('inet:service:platform', {}), {
3736
+ 'doc': 'A parent platform which owns this platform.'}),
3737
+
3738
+ ('status', ('inet:service:object:status', {}), {
3739
+ 'doc': 'The status of the platform.'}),
3740
+
3741
+ ('period', ('ival', {}), {
3742
+ 'doc': 'The period when the platform existed.'}),
3743
+
3744
+ ('creator', ('inet:service:account', {}), {
3745
+ 'doc': 'The service account which created the platform.'}),
3746
+
3747
+ ('remover', ('inet:service:account', {}), {
3748
+ 'doc': 'The service account which removed or decommissioned the platform.'}),
3749
+
3725
3750
  ('provider', ('ou:org', {}), {
3726
3751
  'doc': 'The organization which operates the platform.'}),
3727
3752
 
@@ -3794,8 +3819,12 @@ class InetModule(s_module.CoreModule):
3794
3819
  ('inet:service:account', {}, (
3795
3820
 
3796
3821
  ('user', ('inet:user', {}), {
3822
+ 'alts': ('users',),
3797
3823
  'doc': 'The current user name of the account.'}),
3798
3824
 
3825
+ ('users', ('array', {'type': 'inet:user', 'sorted': True, 'uniq': True}), {
3826
+ 'doc': 'An array of alternate user names for this account.'}),
3827
+
3799
3828
  ('parent', ('inet:service:account', {}), {
3800
3829
  'doc': 'A parent account which owns this account.'}),
3801
3830
 
synapse/models/media.py CHANGED
@@ -35,6 +35,10 @@ class MediaModule(s_module.CoreModule):
35
35
  'disp': {'hint': 'text'},
36
36
  'doc': 'A brief summary of the news item.'}),
37
37
 
38
+ ('body', ('str', {}), {
39
+ 'disp': {'hint': 'text', 'syntax': 'markdown'},
40
+ 'doc': 'The body of the news item.'}),
41
+
38
42
  ('publisher', ('ou:org', {}), {
39
43
  'doc': 'The organization which published the news.'}),
40
44
 
synapse/models/proj.py CHANGED
@@ -166,6 +166,9 @@ class ProjectModule(s_module.CoreModule):
166
166
  ('created', ('time', {}), {
167
167
  'doc': 'The time the comment was added.'}),
168
168
 
169
+ ('ext:creator', ('ps:contact', {}), {
170
+ 'doc': 'The contact information of the creator from an external system.'}),
171
+
169
172
  ('updated', ('time', {'ismax': True}), {
170
173
  'doc': 'The last time the comment was updated.'}),
171
174
 
synapse/models/risk.py CHANGED
@@ -269,6 +269,9 @@ class RiskModule(s_module.CoreModule):
269
269
 
270
270
  (('risk:outage', 'impacted', None), {
271
271
  'doc': 'The outage event impacted the availability of the target node.'}),
272
+
273
+ (('risk:alert', 'about', None), {
274
+ 'doc': 'The alert is about the target node.'}),
272
275
  ),
273
276
  'forms': (
274
277
 
@@ -417,6 +420,9 @@ class RiskModule(s_module.CoreModule):
417
420
  ('type', ('risk:mitigation:type:taxonomy', {}), {
418
421
  'doc': 'A taxonomy type entry for the mitigation.'}),
419
422
 
423
+ ('id', ('str', {'strip': True}), {
424
+ 'doc': 'An identifier for the mitigation.'}),
425
+
420
426
  ('desc', ('str', {}), {
421
427
  'disp': {'hint': 'text'},
422
428
  'doc': 'A description of the mitigation approach for the vulnerability.'}),
@@ -800,6 +806,9 @@ class RiskModule(s_module.CoreModule):
800
806
  ('detected', ('time', {}), {
801
807
  'doc': 'The time the alerted condition was detected.'}),
802
808
 
809
+ ('updated', ('time', {}), {
810
+ 'doc': 'The time the alert was most recently modified.'}),
811
+
803
812
  ('vuln', ('risk:vuln', {}), {
804
813
  'doc': 'The optional vulnerability that the alert indicates.'}),
805
814
 
synapse/models/syn.py CHANGED
@@ -365,6 +365,14 @@ class SynModule(s_module.CoreModule):
365
365
  ('nodedata', ('array', {'type': 'syn:nodedata'}), {
366
366
  'deprecated': True,
367
367
  'doc': 'The list of nodedata that may be added by the command.', 'uniq': True, 'sorted': True, 'ro': True}),
368
+ ('deprecated', ('bool', {}), {
369
+ 'doc': 'Set to true if this command is scheduled to be removed.'}),
370
+ ('deprecated:version', ('it:semver', {}), {
371
+ 'doc': 'The Synapse version when this command will be removed.'}),
372
+ ('deprecated:date', ('time', {}), {
373
+ 'doc': 'The date when this command will be removed.'}),
374
+ ('deprecated:mesg', ('str', {}), {
375
+ 'doc': 'Optional description of this deprecation.'}),
368
376
  )),
369
377
  ),
370
378
  }),)
@@ -75,6 +75,10 @@ class CommonTest(s_t_utils.SynTest):
75
75
  self.ne('15c8a3727942fa01e04d6a7a525666a2', s_common.guid(item))
76
76
  self.eq('15c8a3727942fa01e04d6a7a525666a2', s_common.guid(s_common.flatten(item)))
77
77
 
78
+ item = {'foo': 'bar', 'baz': 10, 'gronk': True, 'hehe': ['ha', 'ha'], 'tupl': (1, 'two', 1.23), 'newp': None}
79
+ self.ne('02efa9b7612f371dbb65a596cd303d9a', s_common.guid(item))
80
+ self.eq('02efa9b7612f371dbb65a596cd303d9a', s_common.guid(s_common.flatten(item)))
81
+
78
82
  def test_common_vertup(self):
79
83
  self.eq(s_common.vertup('1.3.30'), (1, 3, 30))
80
84
  self.true(s_common.vertup('30.40.50') > (9, 0))
@@ -28,8 +28,8 @@ import synapse.lib.version as s_version
28
28
  import synapse.lib.modelrev as s_modelrev
29
29
  import synapse.lib.stormsvc as s_stormsvc
30
30
 
31
- import synapse.tools.backup as s_tools_backup
32
- import synapse.tools.promote as s_tools_promote
31
+ import synapse.tools.service.backup as s_tools_backup
32
+ import synapse.tools.service.promote as s_tools_promote
33
33
 
34
34
  import synapse.tests.utils as s_t_utils
35
35
  from synapse.tests.utils import alist
@@ -6650,6 +6650,57 @@ class CortexBasicTest(s_t_utils.SynTest):
6650
6650
  with self.raises(s_exc.BadPropDef):
6651
6651
  await prox.addTagProp('_blah:blah^blah', ('int', {}), {})
6652
6652
 
6653
+ # Mirrors on newer model versions should not be able to add extended model elements
6654
+ # based on model elements the leader doesn't have
6655
+ async with self.getTestAha() as aha:
6656
+
6657
+ conf = {'aha:provision': await aha.addAhaSvcProv('00.cortex')}
6658
+ core00 = await aha.enter_context(self.getTestCore(conf=conf))
6659
+
6660
+ conf = {'aha:provision': await aha.addAhaSvcProv('01.cortex', {'mirror': 'cortex'})}
6661
+ core01 = await aha.enter_context(self.getTestCore(conf=conf))
6662
+
6663
+ # Add a type directly to the mirror's model to simulate different model version
6664
+ core01.model.addType('_newmodel:type', 'str', {}, {})
6665
+
6666
+ with self.raises(s_exc.NoSuchType):
6667
+ await core01.addType('_test:type', '_newmodel:type', {}, {})
6668
+
6669
+ await core01.sync()
6670
+ self.none(core01.model.type('_test:type'))
6671
+
6672
+ with self.raises(s_exc.NoSuchType):
6673
+ await core01.addUnivProp('_woot', ('_newmodel:type', {}), {})
6674
+
6675
+ await core01.sync()
6676
+ self.none(core01.model.prop('._woot'))
6677
+
6678
+ with self.raises(s_exc.NoSuchType):
6679
+ await core01.addForm('_hehe:haha', '_newmodel:type', {}, {})
6680
+
6681
+ await core01.sync()
6682
+ self.none(core01.model.form('_hehe:haha'))
6683
+
6684
+ with self.raises(s_exc.NoSuchType):
6685
+ await core01.addFormProp('inet:asn', '_newer', ('_newmodel:type', {}), {})
6686
+
6687
+ await core01.sync()
6688
+ self.none(core01.model.prop('inet:asn:_newer'))
6689
+
6690
+ with self.raises(s_exc.NoSuchType):
6691
+ await core01.addTagProp('user', ('_newmodel:type', {}), {})
6692
+
6693
+ await core01.sync()
6694
+ self.none(core01.model.tagprop('user'))
6695
+
6696
+ core01.model.addForm('_newmodel:type', {}, {})
6697
+
6698
+ with self.raises(s_exc.NoSuchForm):
6699
+ await core01.addEdge(('_newmodel:type', '_foo', None), {})
6700
+
6701
+ await core01.sync()
6702
+ self.none(core01.model.edge(('_newmodel:type', '_foo', None)))
6703
+
6653
6704
  async def test_cortex_axon(self):
6654
6705
  async with self.getTestCore() as core:
6655
6706
  # By default, a cortex has a local Axon instance available
@@ -9,7 +9,7 @@ import synapse.exc as s_exc
9
9
  import synapse.common as s_common
10
10
  import synapse.tests.utils as s_t_utils
11
11
 
12
- import synapse.tools.backup as s_tools_backup
12
+ import synapse.tools.service.backup as s_tools_backup
13
13
 
14
14
  import synapse.lib.agenda as s_agenda
15
15
  from synapse.lib.agenda import TimeUnit as s_tu
@@ -38,7 +38,7 @@ import synapse.lib.lmdbslab as s_lmdbslab
38
38
  import synapse.lib.crypto.passwd as s_passwd
39
39
  import synapse.lib.platforms.linux as s_linux
40
40
 
41
- import synapse.tools.backup as s_tools_backup
41
+ import synapse.tools.service.backup as s_tools_backup
42
42
 
43
43
  import synapse.tests.utils as s_t_utils
44
44
 
@@ -9,7 +9,7 @@ import synapse.common as s_common
9
9
  import synapse.lib.certdir as s_certdir
10
10
  import synapse.lib.msgpack as s_msgpack
11
11
  import synapse.tests.utils as s_t_utils
12
- import synapse.tools.genpkg as s_genpkg
12
+ import synapse.tools.storm.pkg.gen as s_genpkg
13
13
 
14
14
  import cryptography.x509 as c_x509
15
15
  import cryptography.exceptions as c_exc
@@ -5,7 +5,7 @@ import aiohttp
5
5
  import aiohttp.client_exceptions as a_exc
6
6
 
7
7
  import synapse.common as s_common
8
- import synapse.tools.backup as s_backup
8
+ import synapse.tools.service.backup as s_backup
9
9
 
10
10
  import synapse.exc as s_exc
11
11
  import synapse.lib.coro as s_coro
@@ -13,7 +13,7 @@ import synapse.lib.layer as s_layer
13
13
  import synapse.lib.msgpack as s_msgpack
14
14
  import synapse.lib.spooled as s_spooled
15
15
 
16
- import synapse.tools.backup as s_tools_backup
16
+ import synapse.tools.service.backup as s_tools_backup
17
17
 
18
18
  import synapse.tests.utils as s_t_utils
19
19
 
@@ -226,6 +226,28 @@ class MultiSlabSeqn(s_t_utils.SynTest):
226
226
  # Iterator exhausted: should have just the cache slab (10) and the tail slab (20)
227
227
  self.len(2, msqn._openslabs)
228
228
 
229
+ async def test_multislabseqn_retnpack(self):
230
+
231
+ with self.getTestDir() as dirn:
232
+
233
+ async with await s_multislabseqn.MultiSlabSeqn.anit(dirn) as msqn:
234
+
235
+ self.eq((0, b'\xa4foo1'), await msqn.addWithPackRetn('foo1'))
236
+ self.eq((1, b'\xa4foo2'), await msqn.addWithPackRetn('foo2'))
237
+ self.eq((0, b'\xa4foo3'), await msqn.addWithPackRetn('foo3', indx=0))
238
+
239
+ await msqn.addWithPackRetn('foo4', indx=10)
240
+ await msqn.rotate()
241
+ await msqn.addWithPackRetn('foo5')
242
+ await msqn.addWithPackRetn('foo6', indx=7)
243
+
244
+ self.true(await msqn.cull(10))
245
+
246
+ with self.raises(s_exc.BadIndxValu):
247
+ await msqn.addWithPackRetn('foo7', indx=7)
248
+
249
+ self.eq((12, b'\xa4foo8'), msqn.tailseqn.addWithPackRetn('foo8'))
250
+
229
251
  async def test_multislabseqn_cull(self):
230
252
 
231
253
  with self.getTestDir() as dirn:
@@ -1,4 +1,5 @@
1
1
  import asyncio
2
+ import contextlib
2
3
  from unittest import mock
3
4
 
4
5
  import synapse.exc as s_exc
@@ -61,7 +62,7 @@ class SampleNexus2(SampleNexus, SampleMixin):
61
62
 
62
63
  class NexusTest(s_t_utils.SynTest):
63
64
 
64
- async def test_nexus(self):
65
+ async def test_nexus_base(self):
65
66
 
66
67
  with self.getTestDir() as dirn:
67
68
  dir1 = s_common.genpath(dirn, 'nexus1')
@@ -111,6 +112,19 @@ class NexusTest(s_t_utils.SynTest):
111
112
  stream.seek(0)
112
113
  self.isin('while replaying log', stream.read())
113
114
 
115
+ nexsindx = nexus1.nexsroot.nexslog.index()
116
+
117
+ async def listen():
118
+ async for item in nexus1.getNexusChanges(nexsindx, wait=True):
119
+ break
120
+ return item
121
+
122
+ task = nexus1.schedCoro(listen())
123
+ self.eq('foo', await nexus1.doathing(eventdict))
124
+ offs, item = await asyncio.wait_for(task, timeout=12)
125
+ self.eq(offs, nexsindx)
126
+ self.eq(item[1], 'thing:doathing')
127
+
114
128
  async def test_nexus_modroot(self):
115
129
 
116
130
  async with self.getTestCell() as cell:
@@ -395,3 +409,30 @@ class NexusTest(s_t_utils.SynTest):
395
409
  await core.nodes('[ it:dev:str=foo ]', opts={'view': forkiden})
396
410
 
397
411
  core.nexsroot._nexskids[layriden] = layr
412
+
413
+ async def test_nexus_iter(self):
414
+ async with self.getTestCell(conf={'nexslog:en': True}) as cell:
415
+ await cell.sync()
416
+
417
+ # Force nexus events to be added while constructing a Window
418
+ orig = s_nexus.NexsRoot.getMirrorWindow
419
+
420
+ @contextlib.asynccontextmanager
421
+ async def slowwindow(self):
422
+ await cell.sync()
423
+ async with orig(self) as wind:
424
+ await cell.sync()
425
+ yield wind
426
+
427
+ items = []
428
+ with mock.patch('synapse.lib.nexus.NexsRoot.getMirrorWindow', slowwindow):
429
+ async with cell.getLocalProxy() as prox:
430
+ async for indx, item in prox.getNexusChanges(0):
431
+ items.append(indx)
432
+ if indx == 2:
433
+ await cell.sync()
434
+ elif indx == 3:
435
+ break
436
+
437
+ # We didn't miss or duplicate items added during window construction
438
+ self.eq([0, 1, 2, 3], items)
@@ -15,7 +15,7 @@ class SlabSeqn(s_t_utils.SynTest):
15
15
  def chk_size(self, seqn):
16
16
  self.eq(seqn.stat()['entries'], seqn.size)
17
17
 
18
- async def test_slab_seqn(self):
18
+ async def test_slab_seqn_base(self):
19
19
 
20
20
  with self.getTestDir() as dirn:
21
21
 
@@ -174,3 +174,32 @@ class SlabSeqn(s_t_utils.SynTest):
174
174
 
175
175
  res = await asyncio.wait_for(genr.__anext__(), timeout=1)
176
176
  self.eq((1, 'bar'), res)
177
+
178
+ async def test_slab_seqn_retnpack(self):
179
+
180
+ with self.getTestDir() as dirn:
181
+ async with await s_lmdbslab.Slab.anit(dirn, map_size=10_0000) as slab:
182
+
183
+ seqn = s_slabseqn.SlabSeqn(slab, 'seqn:test')
184
+
185
+ self.eq((0, b'\xa4foo1'), seqn.addWithPackRetn('foo1'))
186
+ self.eq((1, b'\xa4foo2'), seqn.addWithPackRetn('foo2'))
187
+ self.eq((0, b'\xa4foo3'), seqn.addWithPackRetn('foo3', indx=0))
188
+
189
+ seqn.addWithPackRetn('foo4', indx=10)
190
+ seqn.addWithPackRetn('foo5')
191
+ seqn.addWithPackRetn('foo6', indx=7)
192
+ seqn.addWithPackRetn('foo7', indx=7)
193
+
194
+ self.eq((12, b'\xa4foo8'), seqn.addWithPackRetn('foo8'))
195
+
196
+ valus = [valu for valu in seqn.iter(0)]
197
+ evals = [
198
+ (0, 'foo3',),
199
+ (1, 'foo2'),
200
+ (7, 'foo7'),
201
+ (10, 'foo4'),
202
+ (11, 'foo5'),
203
+ (12, 'foo8'),
204
+ ]
205
+ self.eq(valus, evals)