synapse 2.224.0__py311-none-any.whl → 2.226.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (136) hide show
  1. synapse/axon.py +10 -5
  2. synapse/cortex.py +6 -8
  3. synapse/lib/agenda.py +28 -15
  4. synapse/lib/ast.py +4 -8
  5. synapse/lib/cell.py +1 -1
  6. synapse/lib/const.py +4 -0
  7. synapse/lib/multislabseqn.py +36 -1
  8. synapse/lib/nexus.py +67 -8
  9. synapse/lib/platforms/linux.py +2 -0
  10. synapse/lib/queue.py +4 -1
  11. synapse/lib/rstorm.py +2 -2
  12. synapse/lib/schemas.py +11 -1
  13. synapse/lib/slabseqn.py +28 -0
  14. synapse/lib/storm.py +16 -4
  15. synapse/lib/stormhttp.py +7 -1
  16. synapse/lib/stormlib/aha.py +3 -3
  17. synapse/lib/stormtypes.py +10 -3
  18. synapse/lib/types.py +20 -0
  19. synapse/lib/version.py +2 -2
  20. synapse/models/base.py +3 -0
  21. synapse/models/inet.py +62 -5
  22. synapse/models/infotech.py +18 -0
  23. synapse/models/media.py +4 -0
  24. synapse/models/risk.py +3 -0
  25. synapse/tests/test_cortex.py +115 -2
  26. synapse/tests/test_lib_agenda.py +141 -28
  27. synapse/tests/test_lib_cell.py +1 -1
  28. synapse/tests/test_lib_certdir.py +1 -1
  29. synapse/tests/test_lib_httpapi.py +1 -1
  30. synapse/tests/test_lib_layer.py +1 -1
  31. synapse/tests/test_lib_lmdbslab.py +2 -0
  32. synapse/tests/test_lib_multislabseqn.py +22 -0
  33. synapse/tests/test_lib_nexus.py +42 -1
  34. synapse/tests/test_lib_platforms_linux.py +4 -0
  35. synapse/tests/test_lib_slabseqn.py +30 -1
  36. synapse/tests/test_lib_storm.py +65 -1
  37. synapse/tests/test_lib_stormhttp.py +16 -0
  38. synapse/tests/test_lib_stormlib_aha.py +6 -2
  39. synapse/tests/test_lib_stormlib_oauth.py +1 -1
  40. synapse/tests/test_lib_stormsvc.py +1 -1
  41. synapse/tests/test_lib_trigger.py +1 -1
  42. synapse/tests/test_model_inet.py +37 -0
  43. synapse/tests/test_model_infotech.py +15 -1
  44. synapse/tests/test_model_media.py +4 -1
  45. synapse/tests/test_model_risk.py +2 -0
  46. synapse/tests/test_tools_aha.py +2 -2
  47. synapse/tests/{test_tools_axon2axon.py → test_tools_axon_copy.py} +4 -4
  48. synapse/tests/{test_tools_pullfile.py → test_tools_axon_get.py} +4 -4
  49. synapse/tests/{test_tools_pushfile.py → test_tools_axon_put.py} +7 -7
  50. synapse/tests/{test_tools_csvtool.py → test_tools_cortex_csv.py} +12 -3
  51. synapse/tests/{test_tools_feed.py → test_tools_cortex_feed.py} +2 -2
  52. synapse/tests/{test_tools_apikey.py → test_tools_service_apikey.py} +1 -4
  53. synapse/tests/{test_tools_backup.py → test_tools_service_backup.py} +5 -5
  54. synapse/tests/{test_tools_demote.py → test_tools_service_demote.py} +1 -1
  55. synapse/tests/{test_tools_healthcheck.py → test_tools_service_healthcheck.py} +1 -1
  56. synapse/tests/{test_tools_livebackup.py → test_tools_service_livebackup.py} +1 -1
  57. synapse/tests/{test_tools_modrole.py → test_tools_service_modrole.py} +1 -1
  58. synapse/tests/{test_tools_moduser.py → test_tools_service_moduser.py} +1 -1
  59. synapse/tests/{test_tools_promote.py → test_tools_service_promote.py} +1 -1
  60. synapse/tests/{test_tools_reload.py → test_tools_service_reload.py} +1 -1
  61. synapse/tests/{test_tools_shutdown.py → test_tools_service_shutdown.py} +1 -1
  62. synapse/tests/{test_tools_snapshot.py → test_tools_service_snapshot.py} +1 -1
  63. synapse/tests/{test_tools_storm.py → test_tools_storm_cli.py} +1 -1
  64. synapse/tests/{test_tools_pkgs_gendocs.py → test_tools_storm_pkg_doc.py} +12 -3
  65. synapse/tests/{test_tools_genpkg.py → test_tools_storm_pkg_gen.py} +1 -1
  66. synapse/tests/{test_tools_autodoc.py → test_tools_utils_autodoc.py} +1 -1
  67. synapse/tests/test_tools_utils_changelog.py +454 -0
  68. synapse/tests/{test_tools_easycert.py → test_tools_utils_easycert.py} +48 -46
  69. synapse/tests/{test_tools_guid.py → test_tools_utils_guid.py} +3 -3
  70. synapse/tests/{test_tools_json2mpk.py → test_tools_utils_json2mpk.py} +3 -3
  71. synapse/tests/{test_tools_rstorm.py → test_tools_utils_rstorm.py} +6 -1
  72. synapse/tests/utils.py +15 -1
  73. synapse/tools/aha/mirror.py +1 -1
  74. synapse/tools/apikey.py +4 -83
  75. synapse/tools/autodoc.py +3 -1031
  76. synapse/tools/axon/copy.py +44 -0
  77. synapse/tools/axon/get.py +64 -0
  78. synapse/tools/axon/put.py +122 -0
  79. synapse/tools/axon2axon.py +3 -36
  80. synapse/tools/backup.py +6 -176
  81. synapse/tools/changelog.py +3 -1098
  82. synapse/tools/cortex/csv.py +236 -0
  83. synapse/tools/cortex/feed.py +151 -0
  84. synapse/tools/csvtool.py +3 -227
  85. synapse/tools/demote.py +4 -40
  86. synapse/tools/docker/validate.py +3 -3
  87. synapse/tools/easycert.py +4 -129
  88. synapse/tools/feed.py +3 -140
  89. synapse/tools/genpkg.py +3 -307
  90. synapse/tools/guid.py +7 -6
  91. synapse/tools/healthcheck.py +3 -101
  92. synapse/tools/json2mpk.py +6 -38
  93. synapse/tools/livebackup.py +4 -27
  94. synapse/tools/modrole.py +3 -108
  95. synapse/tools/moduser.py +3 -179
  96. synapse/tools/pkgs/gendocs.py +3 -164
  97. synapse/tools/promote.py +4 -41
  98. synapse/tools/pullfile.py +3 -56
  99. synapse/tools/pushfile.py +3 -114
  100. synapse/tools/reload.py +4 -61
  101. synapse/tools/rstorm.py +3 -26
  102. synapse/tools/service/__init__.py +0 -0
  103. synapse/tools/service/apikey.py +90 -0
  104. synapse/tools/service/backup.py +181 -0
  105. synapse/tools/service/demote.py +47 -0
  106. synapse/tools/service/healthcheck.py +109 -0
  107. synapse/tools/service/livebackup.py +34 -0
  108. synapse/tools/service/modrole.py +116 -0
  109. synapse/tools/service/moduser.py +184 -0
  110. synapse/tools/service/promote.py +48 -0
  111. synapse/tools/service/reload.py +68 -0
  112. synapse/tools/service/shutdown.py +51 -0
  113. synapse/tools/service/snapshot.py +64 -0
  114. synapse/tools/shutdown.py +5 -45
  115. synapse/tools/snapshot.py +4 -57
  116. synapse/tools/storm/__init__.py +0 -0
  117. synapse/tools/storm/__main__.py +5 -0
  118. synapse/tools/{storm.py → storm/_cli.py} +0 -3
  119. synapse/tools/storm/pkg/__init__.py +0 -0
  120. synapse/tools/{pkgs/pandoc_filter.py → storm/pkg/_pandoc_filter.py} +1 -1
  121. synapse/tools/storm/pkg/doc.py +176 -0
  122. synapse/tools/storm/pkg/gen.py +315 -0
  123. synapse/tools/utils/__init__.py +0 -0
  124. synapse/tools/utils/autodoc.py +1040 -0
  125. synapse/tools/utils/changelog.py +1124 -0
  126. synapse/tools/utils/easycert.py +136 -0
  127. synapse/tools/utils/guid.py +11 -0
  128. synapse/tools/utils/json2mpk.py +46 -0
  129. synapse/tools/utils/rstorm.py +35 -0
  130. {synapse-2.224.0.dist-info → synapse-2.226.0.dist-info}/METADATA +1 -1
  131. {synapse-2.224.0.dist-info → synapse-2.226.0.dist-info}/RECORD +135 -106
  132. synapse/tests/test_tools_changelog.py +0 -196
  133. /synapse/tests/{test_tools_axon.py → test_tools_axon_dump_load.py} +0 -0
  134. {synapse-2.224.0.dist-info → synapse-2.226.0.dist-info}/WHEEL +0 -0
  135. {synapse-2.224.0.dist-info → synapse-2.226.0.dist-info}/licenses/LICENSE +0 -0
  136. {synapse-2.224.0.dist-info → synapse-2.226.0.dist-info}/top_level.txt +0 -0
synapse/lib/version.py CHANGED
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
223
223
  ##############################################################################
224
224
  # The following are touched during the release process by bumpversion.
225
225
  # Do not modify these directly.
226
- version = (2, 224, 0)
226
+ version = (2, 226, 0)
227
227
  verstring = '.'.join([str(x) for x in version])
228
- commit = '7c6e619bf5fd9c5801891b489b11bbee7fdbebd0'
228
+ commit = '90a28a430a0b2b1f8cc74af240f55bfcf717b582'
synapse/models/base.py CHANGED
@@ -176,6 +176,9 @@ class BaseModule(s_module.CoreModule):
176
176
  ((None, 'refs', None), {
177
177
  'doc': 'The source node contains a reference to the target node.'}),
178
178
 
179
+ ((None, 'linked', None), {
180
+ 'doc': 'The source node is linked to the target node.'}),
181
+
179
182
  (('meta:source', 'seen', None), {
180
183
  'doc': 'The meta:source observed the target node.'}),
181
184
 
synapse/models/inet.py CHANGED
@@ -1550,13 +1550,23 @@ class InetModule(s_module.CoreModule):
1550
1550
  ('inet:ssl:jarmsample', ('comp', {'fields': (('server', 'inet:server'), ('jarmhash', 'inet:ssl:jarmhash'))}), {
1551
1551
  'doc': 'A JARM hash sample taken from a server.'}),
1552
1552
 
1553
+ ('inet:service:platform:type:taxonomy', ('taxonomy', {}), {
1554
+ 'interfaces': ('meta:taxonomy',),
1555
+ 'doc': 'A service platform type taxonomy.'}),
1556
+
1553
1557
  ('inet:service:platform', ('guid', {}), {
1554
1558
  'doc': 'A network platform which provides services.'}),
1555
1559
 
1560
+ ('inet:service:agent', ('guid', {}), {
1561
+ 'interfaces': ('inet:service:object',),
1562
+ 'template': {'service:base': 'agent'},
1563
+ 'doc': 'An instance of a deployed agent or software integration which is part of the service architecture.'}),
1564
+
1556
1565
  ('inet:service:app', ('guid', {}), {
1557
1566
  'interfaces': ('inet:service:object',),
1558
1567
  'template': {'service:base': 'application'},
1559
- 'doc': 'An application which is part of a service architecture.'}),
1568
+ 'deprecated': True,
1569
+ 'doc': 'Deprecated. Please use inet:service:agent for autonomous agents.'}),
1560
1570
 
1561
1571
  ('inet:service:instance', ('guid', {}), {
1562
1572
  'doc': 'An instance of the platform such as Slack or Discord instances.'}),
@@ -1795,7 +1805,8 @@ class InetModule(s_module.CoreModule):
1795
1805
  'doc': 'The service account which removed or decommissioned the {service:base}.'}),
1796
1806
 
1797
1807
  ('app', ('inet:service:app', {}), {
1798
- 'doc': 'The app which contains the {service:base}.'}),
1808
+ 'deprecated': True,
1809
+ 'doc': 'Deprecated. Objects are no longer scoped to an application or agent.'}),
1799
1810
  ),
1800
1811
  }),
1801
1812
 
@@ -1816,7 +1827,11 @@ class InetModule(s_module.CoreModule):
1816
1827
  'props': (
1817
1828
 
1818
1829
  ('app', ('inet:service:app', {}), {
1819
- 'doc': 'The app which handled the action.'}),
1830
+ 'deprecated': True,
1831
+ 'doc': 'Deprecated. Please use :agent / inet:service:agent.'}),
1832
+
1833
+ ('agent', ('inet:service:agent', {}), {
1834
+ 'doc': 'The service agent which performed the action potentially on behalf of an account.'}),
1820
1835
 
1821
1836
  ('time', ('time', {}), {
1822
1837
  'doc': 'The time that the account initiated the action.'}),
@@ -1851,8 +1866,12 @@ class InetModule(s_module.CoreModule):
1851
1866
  ('client:host', ('it:host', {}), {
1852
1867
  'doc': 'The client host which initiated the action.'}),
1853
1868
 
1869
+ ('client:software', ('it:prod:softver', {}), {
1870
+ 'doc': 'The client software used to initiate the action.'}),
1871
+
1854
1872
  ('client:app', ('inet:service:app', {}), {
1855
- 'doc': 'The client service app which initiated the action.'}),
1873
+ 'deprecated': True,
1874
+ 'doc': 'Deprecated. Please use :client:software.'}),
1856
1875
 
1857
1876
  ('server', ('inet:server', {}), {
1858
1877
  'doc': 'The network address of the server which handled the action.'}),
@@ -3698,6 +3717,8 @@ class InetModule(s_module.CoreModule):
3698
3717
  'ro': True,
3699
3718
  'doc': 'The x509 certificate sent by the client.'})
3700
3719
  )),
3720
+
3721
+ ('inet:service:platform:type:taxonomy', {}, ()),
3701
3722
  ('inet:service:platform', {}, (
3702
3723
 
3703
3724
  ('id', ('str', {'strip': True}), {
@@ -3732,6 +3753,12 @@ class InetModule(s_module.CoreModule):
3732
3753
  'disp': {'hint': 'text'},
3733
3754
  'doc': 'A description of the service platform.'}),
3734
3755
 
3756
+ ('type', ('inet:service:platform:type:taxonomy', {}), {
3757
+ 'doc': 'The type of service platform.'}),
3758
+
3759
+ ('family', ('str', {'onespace': True, 'lower': True}), {
3760
+ 'doc': 'A family designation for use with instanced platforms such as Slack, Discord, or Mastodon.'}),
3761
+
3735
3762
  ('parent', ('inet:service:platform', {}), {
3736
3763
  'doc': 'A parent platform which owns this platform.'}),
3737
3764
 
@@ -3752,6 +3779,9 @@ class InetModule(s_module.CoreModule):
3752
3779
 
3753
3780
  ('provider:name', ('ou:name', {}), {
3754
3781
  'doc': 'The name of the organization which operates the platform.'}),
3782
+
3783
+ ('software', ('it:prod:softver', {}), {
3784
+ 'doc': 'The latest known software version that the platform is running.'}),
3755
3785
  )),
3756
3786
 
3757
3787
  ('inet:service:instance', {}, (
@@ -3791,7 +3821,8 @@ class InetModule(s_module.CoreModule):
3791
3821
  'doc': 'The tenant which contains the instance.'}),
3792
3822
 
3793
3823
  ('app', ('inet:service:app', {}), {
3794
- 'doc': 'The app which contains the instance.'}),
3824
+ 'deprecated': True,
3825
+ 'doc': 'Deprecated. Instances are no longer scoped to applications.'}),
3795
3826
  )),
3796
3827
 
3797
3828
  ('inet:service:app', {}, (
@@ -3816,11 +3847,34 @@ class InetModule(s_module.CoreModule):
3816
3847
  'doc': 'The name of the organization which provides the application.'}),
3817
3848
  )),
3818
3849
 
3850
+ ('inet:service:agent', {}, (
3851
+
3852
+ ('name', ('str', {'lower': True, 'onespace': True}), {
3853
+ 'alts': ('names',),
3854
+ 'doc': 'The name of the service agent instance.'}),
3855
+
3856
+ ('names', ('array', {'type': 'str',
3857
+ 'typeopts': {'onespace': True, 'lower': True},
3858
+ 'sorted': True, 'uniq': True}), {
3859
+ 'doc': 'An array of alternate names for the service agent instance.'}),
3860
+
3861
+ ('desc', ('str', {}), {
3862
+ 'disp': {'hint': 'text'},
3863
+ 'doc': 'A description of the deployed service agent instance.'}),
3864
+
3865
+ ('software', ('it:prod:softver', {}), {
3866
+ 'doc': 'The latest known software version running on the service agent instance.'}),
3867
+ )),
3868
+
3819
3869
  ('inet:service:account', {}, (
3820
3870
 
3821
3871
  ('user', ('inet:user', {}), {
3872
+ 'alts': ('users',),
3822
3873
  'doc': 'The current user name of the account.'}),
3823
3874
 
3875
+ ('users', ('array', {'type': 'inet:user', 'sorted': True, 'uniq': True}), {
3876
+ 'doc': 'An array of alternate user names for this account.'}),
3877
+
3824
3878
  ('parent', ('inet:service:account', {}), {
3825
3879
  'doc': 'A parent account which owns this account.'}),
3826
3880
 
@@ -3907,6 +3961,9 @@ class InetModule(s_module.CoreModule):
3907
3961
 
3908
3962
  ('inet:service:login', {}, (
3909
3963
 
3964
+ ('url', ('inet:url', {}), {
3965
+ 'doc': 'The URL of the login endpoint used for this login attempt.'}),
3966
+
3910
3967
  ('method', ('inet:service:login:method:taxonomy', {}), {
3911
3968
  'doc': 'The type of authentication used for the login. For example "password" or "multifactor.sms".'}),
3912
3969
 
@@ -803,6 +803,9 @@ class ItModule(s_module.CoreModule):
803
803
  ('it:dev:repo:diff', ('guid', {}), {
804
804
  'doc': 'A diff of a file being applied in a single commit.',
805
805
  }),
806
+ ('it:dev:repo:entry', ('guid', {}), {
807
+ 'doc': 'A file included in a repository.',
808
+ }),
806
809
  ('it:dev:repo:issue:label', ('guid', {}), {
807
810
  'interfaces': ('inet:service:object',),
808
811
  'template': {'service:base': 'repository issue label'},
@@ -1119,6 +1122,10 @@ class ItModule(s_module.CoreModule):
1119
1122
  'doc': 'The YARA rule is intended for use in detecting the target node.'}),
1120
1123
  (('it:dev:repo', 'has', 'inet:url'), {
1121
1124
  'doc': 'The repo has content hosted at the URL.'}),
1125
+ (('it:dev:repo:commit', 'has', 'it:dev:repo:entry'), {
1126
+ 'doc': 'The file entry is present in the commit version of the repository.'}),
1127
+ (('it:log:event', 'about', None), {
1128
+ 'doc': 'The it:log:event is about the target node.'}),
1122
1129
  ),
1123
1130
  'forms': (
1124
1131
  ('it:hostname', {}, ()),
@@ -2001,6 +2008,17 @@ class ItModule(s_module.CoreModule):
2001
2008
  'doc': 'The URL where the diff is hosted.'}),
2002
2009
  )),
2003
2010
 
2011
+ ('it:dev:repo:entry', {}, (
2012
+
2013
+ ('repo', ('it:dev:repo', {}), {
2014
+ 'doc': 'The repository which contains the file.'}),
2015
+
2016
+ ('file', ('file:bytes', {}), {
2017
+ 'doc': 'The file which the repository contains.'}),
2018
+
2019
+ ('path', ('file:path', {}), {
2020
+ 'doc': 'The path to the file in the repository.'}),
2021
+ )),
2004
2022
  ('it:dev:repo:issue', {}, (
2005
2023
 
2006
2024
  ('repo', ('it:dev:repo', {}), {
synapse/models/media.py CHANGED
@@ -35,6 +35,10 @@ class MediaModule(s_module.CoreModule):
35
35
  'disp': {'hint': 'text'},
36
36
  'doc': 'A brief summary of the news item.'}),
37
37
 
38
+ ('body', ('str', {}), {
39
+ 'disp': {'hint': 'text', 'syntax': 'markdown'},
40
+ 'doc': 'The body of the news item.'}),
41
+
38
42
  ('publisher', ('ou:org', {}), {
39
43
  'doc': 'The organization which published the news.'}),
40
44
 
synapse/models/risk.py CHANGED
@@ -420,6 +420,9 @@ class RiskModule(s_module.CoreModule):
420
420
  ('type', ('risk:mitigation:type:taxonomy', {}), {
421
421
  'doc': 'A taxonomy type entry for the mitigation.'}),
422
422
 
423
+ ('id', ('str', {'strip': True}), {
424
+ 'doc': 'An identifier for the mitigation.'}),
425
+
423
426
  ('desc', ('str', {}), {
424
427
  'disp': {'hint': 'text'},
425
428
  'doc': 'A description of the mitigation approach for the vulnerability.'}),
@@ -28,8 +28,8 @@ import synapse.lib.version as s_version
28
28
  import synapse.lib.modelrev as s_modelrev
29
29
  import synapse.lib.stormsvc as s_stormsvc
30
30
 
31
- import synapse.tools.backup as s_tools_backup
32
- import synapse.tools.promote as s_tools_promote
31
+ import synapse.tools.service.backup as s_tools_backup
32
+ import synapse.tools.service.promote as s_tools_promote
33
33
 
34
34
  import synapse.tests.utils as s_t_utils
35
35
  from synapse.tests.utils import alist
@@ -9093,3 +9093,116 @@ class CortexBasicTest(s_t_utils.SynTest):
9093
9093
  async with self.getRegrCore('2.213.0-queue-authgates') as core:
9094
9094
  self.nn(await core.getAuthGate('queue:stillhere'))
9095
9095
  self.none(await core.getAuthGate('queue:authtest'))
9096
+
9097
+ async def test_cortex_prop_copy(self):
9098
+ async with self.getTestCore() as core:
9099
+ q = '[test:arrayprop=(ap0,) :strs=(foo, bar, baz)]'
9100
+ self.len(1, await core.nodes(q))
9101
+
9102
+ q = 'test:arrayprop=(ap0,) $l=:strs $r=$l.rem(baz) return(($r, $l))'
9103
+ valu = await core.callStorm(q)
9104
+ self.true(valu[0])
9105
+ self.sorteq(valu[1], ['foo', 'bar'])
9106
+
9107
+ # modifying the property value shouldn't update the node
9108
+ nodes = await core.nodes('test:arrayprop=(ap0,) $l=:strs $l.rem(baz)')
9109
+ self.len(1, nodes)
9110
+ self.sorteq(nodes[0].get('strs'), ['foo', 'bar', 'baz'])
9111
+
9112
+ data = {
9113
+ 'str': 'strval',
9114
+ 'int': 1,
9115
+ 'dict': {'dictkey': 'dictval'},
9116
+ 'list': ('listval0', 'listval1'),
9117
+ 'tuple': ('tupleval0', 'tupleval1'),
9118
+ }
9119
+
9120
+ opts = {
9121
+ 'vars': {
9122
+ 'data': data,
9123
+ }
9124
+ }
9125
+ q = '''
9126
+ [ test:guid=(d0,)
9127
+ :data=$data
9128
+ :comp=(1, foo)
9129
+ :mutcomp=(foo, (1, 2, 3))
9130
+ ]
9131
+ '''
9132
+ self.len(1, await core.nodes(q, opts=opts))
9133
+
9134
+ q = '''
9135
+ test:guid=(d0,)
9136
+ $d=:data
9137
+ $d.list.rem(listval0)
9138
+ $d.str = foo
9139
+ $d.int = ($d.int + 1)
9140
+ $d.dict.foo = bar
9141
+ $d.tuple.append(tupleval2)
9142
+ return($d)
9143
+ '''
9144
+ valu = await core.callStorm(q)
9145
+ self.eq(valu, {
9146
+ 'str': 'foo',
9147
+ 'int': 2,
9148
+ 'dict': {'dictkey': 'dictval', 'foo': 'bar'},
9149
+ 'list': ('listval1',),
9150
+ 'tuple': ('tupleval0', 'tupleval1', 'tupleval2'),
9151
+ })
9152
+
9153
+ # modifying the property value shouldn't update the node
9154
+ q = '''
9155
+ test:guid=(d0,)
9156
+ $d=:data
9157
+ $d.dict = $lib.undef
9158
+ '''
9159
+ nodes = await core.nodes(q)
9160
+ self.len(1, nodes)
9161
+ self.eq(nodes[0].get('data')['dict'], {'dictkey': 'dictval'})
9162
+
9163
+ q = '''
9164
+ test:guid=(d0,)
9165
+ $c=:comp
9166
+ $c.rem((1))
9167
+ $m=:mutcomp
9168
+ $m.1.rem((3))
9169
+ return(($c, :comp, $m, :mutcomp))
9170
+ '''
9171
+ valu = await core.callStorm(q)
9172
+ self.eq(valu, (
9173
+ (1, 'foo'),
9174
+ (1, 'foo'),
9175
+ ('foo', (1, 2)),
9176
+ ('foo', (1, 2, 3)),
9177
+ ))
9178
+
9179
+ # Nodeprops could have mutable types in them so make sure modifying
9180
+ # them doesn't cause modifications to the node
9181
+ q = '''
9182
+ $data = { test:guid=(d0,) return(:data) }
9183
+ [ test:str=foobar :baz=(test:guid:data, $data) ]
9184
+ ($prop, $valu) = :baz
9185
+ $valu.list.rem(listval0)
9186
+ return((:baz, $valu))
9187
+ '''
9188
+ valu = await core.callStorm(q)
9189
+
9190
+ exp = {
9191
+ 'str': 'strval',
9192
+ 'int': 1,
9193
+ 'dict': {'dictkey': 'dictval'},
9194
+ 'list': ('listval1',),
9195
+ 'tuple': ('tupleval0', 'tupleval1'),
9196
+ }
9197
+
9198
+ self.eq(valu, (('test:guid:data', data), exp))
9199
+
9200
+ # Make sure $node.props aren't modifiable either
9201
+ nodes = await core.nodes('test:str=foobar $node.props.baz.1.list.rem(listval0)')
9202
+ self.len(1, nodes)
9203
+ self.eq(nodes[0].get('baz'), ('test:guid:data', data))
9204
+
9205
+ # Dereferencing mutable types from $node.props should
9206
+ # return mutable instances without mutating the original prop valu
9207
+ valu = await core.callStorm('test:str=foobar ($prop, $valu) = :baz $valu.list.rem(listval0) return((:baz, $valu))')
9208
+ self.eq(valu, (('test:guid:data', data), exp))
@@ -9,7 +9,7 @@ import synapse.exc as s_exc
9
9
  import synapse.common as s_common
10
10
  import synapse.tests.utils as s_t_utils
11
11
 
12
- import synapse.tools.backup as s_tools_backup
12
+ import synapse.tools.service.backup as s_tools_backup
13
13
 
14
14
  import synapse.lib.agenda as s_agenda
15
15
  from synapse.lib.agenda import TimeUnit as s_tu
@@ -295,7 +295,7 @@ class AgendaTest(s_t_utils.SynTest):
295
295
 
296
296
  # Cancel the Wednesday/Friday appt
297
297
  await agenda.delete(guid)
298
- await self.asyncraises(s_exc.NoSuchIden, agenda.delete(b'1234'))
298
+ await self.asyncraises(s_exc.NoSuchIden, agenda.delete('1234'))
299
299
 
300
300
  # Then Dec 25
301
301
  unixtime = datetime.datetime(year=2018, month=12, day=25, hour=10, minute=16, tzinfo=tz.utc).timestamp()
@@ -1205,32 +1205,6 @@ class AgendaTest(s_t_utils.SynTest):
1205
1205
  self.false(cron01[0].get('isrunning'))
1206
1206
  self.eq(cron01[0].get('lasterrs')[0], 'aborted')
1207
1207
 
1208
- async def test_agenda_clear_running_none_nexttime(self):
1209
-
1210
- async with self.getTestCore() as core:
1211
-
1212
- cdef = {
1213
- 'creator': core.auth.rootuser.iden,
1214
- 'iden': s_common.guid(),
1215
- 'storm': '$lib.log.info("test")',
1216
- 'reqs': {},
1217
- 'incunit': 'minute',
1218
- 'incvals': 1
1219
- }
1220
- await core.addCronJob(cdef)
1221
-
1222
- appt = core.agenda.appts[cdef['iden']]
1223
- self.true(appt in core.agenda.apptheap)
1224
-
1225
- appt.isrunning = True
1226
- appt.nexttime = None
1227
-
1228
- await core.agenda.clearRunningStatus()
1229
- self.false(appt in core.agenda.apptheap)
1230
-
1231
- crons = await core.callStorm('return($lib.cron.list())')
1232
- self.len(1, crons)
1233
-
1234
1208
  async def test_agenda_lasterrs(self):
1235
1209
 
1236
1210
  async with self.getTestCore() as core:
@@ -1262,3 +1236,142 @@ class AgendaTest(s_t_utils.SynTest):
1262
1236
  appt = await core.agenda.get('test')
1263
1237
  self.true(isinstance(appt.lasterrs, list))
1264
1238
  self.eq(appt.lasterrs, ['error1', 'error2'])
1239
+
1240
+ async def test_cron_at_mirror_cleanup(self):
1241
+
1242
+ with self.getTestDir() as dirn:
1243
+
1244
+ dirn00 = s_common.genpath(dirn, 'core00')
1245
+ dirn01 = s_common.genpath(dirn, 'core01')
1246
+
1247
+ async with self.getTestAha() as aha:
1248
+
1249
+ conf = {'aha:provision': await aha.addAhaSvcProv('00.cortex')}
1250
+ core00 = await aha.enter_context(self.getTestCore(conf=conf, dirn=dirn00))
1251
+
1252
+ conf = {'aha:provision': await aha.addAhaSvcProv('01.cortex', {'mirror': 'cortex'})}
1253
+ core01 = await aha.enter_context(self.getTestCore(conf=conf, dirn=dirn01))
1254
+
1255
+ msgs = await core00.stormlist('cron.at --minute +1 { $lib.log.info(cronran) }')
1256
+ await core01.sync()
1257
+
1258
+ with self.getAsyncLoggerStream('synapse.storm.log', 'cronran') as stream:
1259
+ core00.agenda._addTickOff(60)
1260
+ self.true(await stream.wait(timeout=12))
1261
+
1262
+ await core01.sync()
1263
+
1264
+ # appt.recs should be cleared on both leader and mirror
1265
+ appt = list(core00.agenda.appts.values())[0]
1266
+ self.len(0, appt.recs)
1267
+
1268
+ appt = list(core01.agenda.appts.values())[0]
1269
+ self.len(0, appt.recs)
1270
+
1271
+ # apptheap should also now be empty on both
1272
+ self.len(0, core00.agenda.apptheap)
1273
+ self.len(0, core01.agenda.apptheap)
1274
+
1275
+ await core00.fini()
1276
+ core00 = await aha.enter_context(self.getTestCore(dirn=dirn00))
1277
+
1278
+ await core01.fini()
1279
+ core01 = await aha.enter_context(self.getTestCore(dirn=dirn01))
1280
+
1281
+ # Job still exists on leader and mirror, wasn't removed due to being invalid
1282
+ cron00 = await core00.listCronJobs()
1283
+ cron01 = await core01.listCronJobs()
1284
+ self.len(1, cron00)
1285
+ self.eq(cron00, cron01)
1286
+
1287
+ await core00.delCronJob(cron00[0]['iden'])
1288
+
1289
+ # Add a job that is past due that will be deleted on startup
1290
+ xmas = {'dayofmonth': 25, 'month': 12, 'year': 2099}
1291
+ cdef = {'creator': core00.auth.rootuser.iden,
1292
+ 'storm': '#happyholidays',
1293
+ 'reqs': (xmas,)}
1294
+ guid = s_common.guid()
1295
+ cdef['iden'] = guid
1296
+
1297
+ await core00.addCronJob(cdef)
1298
+ await core01.sync()
1299
+
1300
+ # Move the year back to the past
1301
+ apptdef = core01.agenda.apptdefs.get(guid)
1302
+ apptdef['recs'][0][0]['year'] = 1999
1303
+
1304
+ core00.agenda.apptdefs.set(guid, apptdef)
1305
+ core01.agenda.apptdefs.set(guid, apptdef)
1306
+
1307
+ with self.getAsyncLoggerStream('synapse.lib.agenda', 'This appointment will be removed') as stream:
1308
+ await core01.fini()
1309
+ core01 = await aha.enter_context(self.getTestCore(dirn=dirn01))
1310
+ self.true(await stream.wait(timeout=12))
1311
+
1312
+ # Mirror warns about the invalid appointment but does not remove it
1313
+ self.nn(core01.agenda.apptdefs.get(guid))
1314
+ self.nn(await core01.getAuthGate(guid))
1315
+
1316
+ with self.getAsyncLoggerStream('synapse.lib.agenda', 'Removing invalid appointment') as stream:
1317
+ await core00.fini()
1318
+ core00 = await aha.enter_context(self.getTestCore(dirn=dirn00))
1319
+ self.true(await stream.wait(timeout=12))
1320
+
1321
+ await core01.sync()
1322
+
1323
+ # Leader removes the appointment via delCronJob which cleans up properly
1324
+ self.none(core00.agenda.apptdefs.get(guid))
1325
+ self.none(core01.agenda.apptdefs.get(guid))
1326
+
1327
+ self.none(await core00.getAuthGate(guid))
1328
+ self.none(await core01.getAuthGate(guid))
1329
+
1330
+ # Add a job with iden mismatch for coverage
1331
+ cdef = {'creator': core00.auth.rootuser.iden,
1332
+ 'storm': '#happyholidays',
1333
+ 'reqs': (xmas,)}
1334
+ guid = s_common.guid()
1335
+ cdef['iden'] = guid
1336
+
1337
+ await core00.addCronJob(cdef)
1338
+ await core01.sync()
1339
+
1340
+ apptdef = core01.agenda.apptdefs.get(guid)
1341
+ apptdef['iden'] = s_common.guid()
1342
+
1343
+ core00.agenda.apptdefs.set(guid, apptdef)
1344
+ core01.agenda.apptdefs.set(guid, apptdef)
1345
+
1346
+ with self.getAsyncLoggerStream('synapse.lib.agenda', 'Removing invalid appointment') as stream:
1347
+ await core00.fini()
1348
+ core00 = await aha.enter_context(self.getTestCore(dirn=dirn00))
1349
+
1350
+ await core01.fini()
1351
+ core01 = await aha.enter_context(self.getTestCore(dirn=dirn01))
1352
+ self.true(await stream.wait(timeout=12))
1353
+
1354
+ await core01.sync()
1355
+
1356
+ self.none(core00.agenda.apptdefs.get(guid))
1357
+ self.none(core01.agenda.apptdefs.get(guid))
1358
+
1359
+ with self.getAsyncLoggerStream('synapse.storm.log', 'I AM A ERROR') as stream:
1360
+ q = "cron.at --now ${ while((true)) { $lib.log.error('I AM A ERROR') $lib.time.sleep(6) } }"
1361
+ await core00.nodes(q)
1362
+ self.true(await stream.wait(timeout=12))
1363
+ await core01.sync()
1364
+
1365
+ await core01.promote(graceful=True)
1366
+ await core00.sync()
1367
+
1368
+ cron00 = await core00.listCronJobs()
1369
+ cron01 = await core01.listCronJobs()
1370
+
1371
+ self.len(1, cron00)
1372
+ self.false(cron00[0].get('isrunning'))
1373
+ self.eq(cron00[0].get('lasterrs')[0], 'aborted')
1374
+ self.eq(cron00, cron01)
1375
+
1376
+ self.len(0, core00.agenda.apptheap)
1377
+ self.len(0, core01.agenda.apptheap)
@@ -38,7 +38,7 @@ import synapse.lib.lmdbslab as s_lmdbslab
38
38
  import synapse.lib.crypto.passwd as s_passwd
39
39
  import synapse.lib.platforms.linux as s_linux
40
40
 
41
- import synapse.tools.backup as s_tools_backup
41
+ import synapse.tools.service.backup as s_tools_backup
42
42
 
43
43
  import synapse.tests.utils as s_t_utils
44
44
 
@@ -9,7 +9,7 @@ import synapse.common as s_common
9
9
  import synapse.lib.certdir as s_certdir
10
10
  import synapse.lib.msgpack as s_msgpack
11
11
  import synapse.tests.utils as s_t_utils
12
- import synapse.tools.genpkg as s_genpkg
12
+ import synapse.tools.storm.pkg.gen as s_genpkg
13
13
 
14
14
  import cryptography.x509 as c_x509
15
15
  import cryptography.exceptions as c_exc
@@ -5,7 +5,7 @@ import aiohttp
5
5
  import aiohttp.client_exceptions as a_exc
6
6
 
7
7
  import synapse.common as s_common
8
- import synapse.tools.backup as s_backup
8
+ import synapse.tools.service.backup as s_backup
9
9
 
10
10
  import synapse.exc as s_exc
11
11
  import synapse.lib.coro as s_coro
@@ -13,7 +13,7 @@ import synapse.lib.layer as s_layer
13
13
  import synapse.lib.msgpack as s_msgpack
14
14
  import synapse.lib.spooled as s_spooled
15
15
 
16
- import synapse.tools.backup as s_tools_backup
16
+ import synapse.tools.service.backup as s_tools_backup
17
17
 
18
18
  import synapse.tests.utils as s_t_utils
19
19
 
@@ -369,7 +369,9 @@ class LmdbSlabTest(s_t_utils.SynTest):
369
369
  'vm.dirty_expire_centisecs',
370
370
  'vm.dirty_writeback_centisecs',
371
371
  'vm.dirty_background_ratio',
372
+ 'vm.dirty_background_bytes',
372
373
  'vm.dirty_ratio',
374
+ 'vm.dirty_bytes',
373
375
  ], msgs[0].get('sysctls', {}).keys())
374
376
 
375
377
  async def test_lmdbslab_commit_over_max_xactops(self):
@@ -226,6 +226,28 @@ class MultiSlabSeqn(s_t_utils.SynTest):
226
226
  # Iterator exhausted: should have just the cache slab (10) and the tail slab (20)
227
227
  self.len(2, msqn._openslabs)
228
228
 
229
+ async def test_multislabseqn_retnpack(self):
230
+
231
+ with self.getTestDir() as dirn:
232
+
233
+ async with await s_multislabseqn.MultiSlabSeqn.anit(dirn) as msqn:
234
+
235
+ self.eq((0, b'\xa4foo1'), await msqn.addWithPackRetn('foo1'))
236
+ self.eq((1, b'\xa4foo2'), await msqn.addWithPackRetn('foo2'))
237
+ self.eq((0, b'\xa4foo3'), await msqn.addWithPackRetn('foo3', indx=0))
238
+
239
+ await msqn.addWithPackRetn('foo4', indx=10)
240
+ await msqn.rotate()
241
+ await msqn.addWithPackRetn('foo5')
242
+ await msqn.addWithPackRetn('foo6', indx=7)
243
+
244
+ self.true(await msqn.cull(10))
245
+
246
+ with self.raises(s_exc.BadIndxValu):
247
+ await msqn.addWithPackRetn('foo7', indx=7)
248
+
249
+ self.eq((12, b'\xa4foo8'), msqn.tailseqn.addWithPackRetn('foo8'))
250
+
229
251
  async def test_multislabseqn_cull(self):
230
252
 
231
253
  with self.getTestDir() as dirn: