synapse 2.189.0__py311-none-any.whl → 2.191.0__py311-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of synapse might be problematic. Click here for more details.

Files changed (43) hide show
  1. synapse/common.py +9 -0
  2. synapse/cortex.py +26 -5
  3. synapse/datamodel.py +8 -1
  4. synapse/lib/cell.py +7 -7
  5. synapse/lib/msgpack.py +10 -3
  6. synapse/lib/storm.py +5 -1
  7. synapse/lib/stormhttp.py +3 -10
  8. synapse/lib/stormlib/model.py +37 -0
  9. synapse/lib/stormtypes.py +27 -17
  10. synapse/lib/version.py +2 -2
  11. synapse/lib/view.py +2 -5
  12. synapse/models/auth.py +2 -1
  13. synapse/models/base.py +20 -0
  14. synapse/models/crypto.py +5 -2
  15. synapse/models/economic.py +45 -11
  16. synapse/models/inet.py +78 -21
  17. synapse/models/person.py +11 -4
  18. synapse/models/risk.py +6 -0
  19. synapse/models/syn.py +22 -12
  20. synapse/models/telco.py +3 -1
  21. synapse/tests/test_cortex.py +73 -31
  22. synapse/tests/test_lib_agenda.py +1 -6
  23. synapse/tests/test_lib_cell.py +28 -4
  24. synapse/tests/test_lib_httpapi.py +2 -4
  25. synapse/tests/test_lib_lmdbslab.py +1 -4
  26. synapse/tests/test_lib_stormhttp.py +9 -0
  27. synapse/tests/test_lib_stormlib_cortex.py +1 -3
  28. synapse/tests/test_lib_stormlib_log.py +1 -6
  29. synapse/tests/test_lib_stormlib_model.py +28 -0
  30. synapse/tests/test_lib_stormtypes.py +17 -3
  31. synapse/tests/test_lib_trigger.py +2 -3
  32. synapse/tests/test_model_base.py +12 -2
  33. synapse/tests/test_model_inet.py +23 -0
  34. synapse/tests/test_model_person.py +2 -0
  35. synapse/tests/test_model_risk.py +5 -0
  36. synapse/tests/test_model_syn.py +198 -0
  37. synapse/tests/test_utils.py +23 -4
  38. synapse/tests/utils.py +39 -5
  39. {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/METADATA +5 -5
  40. {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/RECORD +43 -43
  41. {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/LICENSE +0 -0
  42. {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/WHEEL +0 -0
  43. {synapse-2.189.0.dist-info → synapse-2.191.0.dist-info}/top_level.txt +0 -0
synapse/models/inet.py CHANGED
@@ -1530,7 +1530,8 @@ class InetModule(s_module.CoreModule):
1530
1530
  'doc': 'An object status enumeration.'}),
1531
1531
 
1532
1532
  ('inet:service:account', ('guid', {}), {
1533
- 'interfaces': ('inet:service:object',),
1533
+ 'interfaces': ('inet:service:subscriber',),
1534
+ 'template': {'service:base': 'account'},
1534
1535
  'doc': 'An account within a service platform. Accounts may be instance specific.'}),
1535
1536
 
1536
1537
  ('inet:service:relationship:type:taxonomy', ('taxonomy', {}), {
@@ -1539,6 +1540,7 @@ class InetModule(s_module.CoreModule):
1539
1540
 
1540
1541
  ('inet:service:relationship', ('guid', {}), {
1541
1542
  'interfaces': ('inet:service:object',),
1543
+ 'template': {'service:base': 'relationship'},
1542
1544
  'doc': 'A relationship between two service objects.'}),
1543
1545
 
1544
1546
  ('inet:service:permission:type:taxonomy', ('taxonomy', {}), {
@@ -1547,10 +1549,12 @@ class InetModule(s_module.CoreModule):
1547
1549
 
1548
1550
  ('inet:service:permission', ('guid', {}), {
1549
1551
  'interfaces': ('inet:service:object',),
1552
+ 'template': {'service:base': 'permission'},
1550
1553
  'doc': 'A permission which may be granted to a service account or role.'}),
1551
1554
 
1552
1555
  ('inet:service:rule', ('guid', {}), {
1553
1556
  'interfaces': ('inet:service:object',),
1557
+ 'template': {'service:base': 'rule'},
1554
1558
  'doc': 'A rule which grants or denies a permission to a service account or role.'}),
1555
1559
 
1556
1560
  ('inet:service:login', ('guid', {}), {
@@ -1563,26 +1567,32 @@ class InetModule(s_module.CoreModule):
1563
1567
 
1564
1568
  ('inet:service:session', ('guid', {}), {
1565
1569
  'interfaces': ('inet:service:object',),
1570
+ 'template': {'service:base': 'session'},
1566
1571
  'doc': 'An authenticated session.'}),
1567
1572
 
1568
1573
  ('inet:service:group', ('guid', {}), {
1569
1574
  'interfaces': ('inet:service:object',),
1575
+ 'template': {'service:base': 'group'},
1570
1576
  'doc': 'A group or role which contains member accounts.'}),
1571
1577
 
1572
1578
  ('inet:service:group:member', ('guid', {}), {
1573
1579
  'interfaces': ('inet:service:object',),
1580
+ 'template': {'service:base': 'group membership'},
1574
1581
  'doc': 'Represents a service account being a member of a group.'}),
1575
1582
 
1576
1583
  ('inet:service:channel', ('guid', {}), {
1577
1584
  'interfaces': ('inet:service:object',),
1585
+ 'template': {'service:base': 'channel'},
1578
1586
  'doc': 'A channel used to distribute messages.'}),
1579
1587
 
1580
1588
  ('inet:service:thread', ('guid', {}), {
1581
1589
  'interfaces': ('inet:service:object',),
1590
+ 'template': {'service:base': 'thread'},
1582
1591
  'doc': 'A message thread.'}),
1583
1592
 
1584
1593
  ('inet:service:channel:member', ('guid', {}), {
1585
1594
  'interfaces': ('inet:service:object',),
1595
+ 'template': {'service:base': 'channel membership'},
1586
1596
  'doc': 'Represents a service account being a member of a channel.'}),
1587
1597
 
1588
1598
  ('inet:service:message', ('guid', {}), {
@@ -1601,26 +1611,47 @@ class InetModule(s_module.CoreModule):
1601
1611
 
1602
1612
  ('inet:service:emote', ('guid', {}), {
1603
1613
  'interfaces': ('inet:service:object',),
1614
+ 'template': {'service:base': 'emote'},
1604
1615
  'doc': 'An emote or reaction by an account.'}),
1605
1616
 
1606
1617
  ('inet:service:access', ('guid', {}), {
1607
1618
  'interfaces': ('inet:service:action',),
1608
1619
  'doc': 'Represents a user access request to a service resource.'}),
1609
1620
 
1621
+ ('inet:service:tenant', ('guid', {}), {
1622
+ 'interfaces': ('inet:service:subscriber',),
1623
+ 'template': {'service:base': 'tenant'},
1624
+ 'doc': 'A tenant which groups accounts and instances.'}),
1625
+
1626
+ ('inet:service:subscription:level:taxonomy', ('taxonomy', {}), {
1627
+ 'interfaces': ('meta:taxonomy',),
1628
+ 'doc': 'A taxonomy of platform specific subscription levels.'}),
1629
+
1630
+ ('inet:service:subscription', ('guid', {}), {
1631
+ 'interfaces': ('inet:service:object',),
1632
+ 'template': {'service:base': 'subscription'},
1633
+ 'doc': 'A subscription to a service platform or instance.'}),
1634
+
1635
+ ('inet:service:subscriber', ('ndef', {'interface': 'inet:service:subscriber'}), {
1636
+ 'doc': 'A node which may subscribe to a service subscription.'}),
1637
+
1610
1638
  ('inet:service:resource:type:taxonomy', ('taxonomy', {}), {
1611
1639
  'interfaces': ('meta:taxonomy',),
1612
1640
  'doc': 'A taxonomy of inet service resource types.'}),
1613
1641
 
1614
1642
  ('inet:service:resource', ('guid', {}), {
1615
1643
  'interfaces': ('inet:service:object',),
1644
+ 'template': {'service:base': 'resource'},
1616
1645
  'doc': 'A generic resource provided by the service architecture.'}),
1617
1646
 
1618
1647
  ('inet:service:bucket', ('guid', {}), {
1619
1648
  'interfaces': ('inet:service:object',),
1649
+ 'template': {'service:base': 'bucket'},
1620
1650
  'doc': 'A file/blob storage object within a service architecture.'}),
1621
1651
 
1622
1652
  ('inet:service:bucket:item', ('guid', {}), {
1623
1653
  'interfaces': ('inet:service:object',),
1654
+ 'template': {'service:base': 'bucket item'},
1624
1655
  'doc': 'An individual file stored within a bucket.'}),
1625
1656
 
1626
1657
  ('inet:tls:handshake', ('guid', {}), {
@@ -1676,16 +1707,17 @@ class InetModule(s_module.CoreModule):
1676
1707
 
1677
1708
  ('inet:service:base', {
1678
1709
  'doc': 'Properties common to most forms within a service platform.',
1710
+ 'template': {'service:base': 'node'},
1679
1711
  'props': (
1680
1712
 
1681
1713
  ('id', ('str', {'strip': True}), {
1682
- 'doc': 'A platform specific ID.'}),
1714
+ 'doc': 'A platform specific ID which identifies the {service:base}.'}),
1683
1715
 
1684
1716
  ('platform', ('inet:service:platform', {}), {
1685
- 'doc': 'The platform which defines the node.'}),
1717
+ 'doc': 'The platform which defines the {service:base}.'}),
1686
1718
 
1687
1719
  ('instance', ('inet:service:instance', {}), {
1688
- 'doc': 'The platform instance which defines the node.'}),
1720
+ 'doc': 'The platform instance which defines the {service:base}.'}),
1689
1721
  ),
1690
1722
  }),
1691
1723
 
@@ -1693,23 +1725,34 @@ class InetModule(s_module.CoreModule):
1693
1725
 
1694
1726
  'doc': 'Properties common to objects within a service platform.',
1695
1727
  'interfaces': ('inet:service:base',),
1728
+ 'template': {'service:base': 'object'},
1696
1729
  'props': (
1697
1730
 
1698
1731
  ('status', ('inet:service:object:status', {}), {
1699
- 'doc': 'The status of this object.'}),
1732
+ 'doc': 'The status of the {service:base}.'}),
1700
1733
 
1701
1734
  ('period', ('ival', {}), {
1702
- 'doc': 'The period when the object existed.'}),
1735
+ 'doc': 'The period when the {service:base} existed.'}),
1703
1736
 
1704
1737
  ('creator', ('inet:service:account', {}), {
1705
- 'doc': 'The service account which created the object.'}),
1738
+ 'doc': 'The service account which created the {service:base}.'}),
1706
1739
 
1707
1740
  ('remover', ('inet:service:account', {}), {
1708
- 'doc': 'The service account which removed or decommissioned the object.'}),
1741
+ 'doc': 'The service account which removed or decommissioned the {service:base}.'}),
1709
1742
 
1710
1743
  ),
1711
1744
  }),
1712
1745
 
1746
+ ('inet:service:subscriber', {
1747
+ 'doc': 'Properties common to the nodes which subscribe to services.',
1748
+ 'interfaces': ('inet:service:object',),
1749
+ 'template': {'service:base': 'subscriber'},
1750
+ 'props': (
1751
+ ('profile', ('ps:contact', {}), {
1752
+ 'doc': 'The primary contact information for the {service:base}.'}),
1753
+ ),
1754
+ }),
1755
+
1713
1756
  ('inet:service:action', {
1714
1757
 
1715
1758
  'doc': 'Properties common to events within a service platform.',
@@ -3590,21 +3633,21 @@ class InetModule(s_module.CoreModule):
3590
3633
 
3591
3634
  ('owner', ('inet:service:account', {}), {
3592
3635
  'doc': 'The service account which owns the instance.'}),
3636
+
3637
+ ('tenant', ('inet:service:tenant', {}), {
3638
+ 'doc': 'The tenant which contains the instance.'}),
3593
3639
  )),
3594
3640
 
3595
3641
  ('inet:service:account', {}, (
3596
3642
 
3597
- ('id', ('str', {'strip': True}), {
3598
- 'doc': 'A platform specific ID used to identify the account.'}),
3599
-
3600
3643
  ('user', ('inet:user', {}), {
3601
3644
  'doc': 'The current user name of the account.'}),
3602
3645
 
3603
3646
  ('email', ('inet:email', {}), {
3604
3647
  'doc': 'The current email address associated with the account.'}),
3605
3648
 
3606
- ('profile', ('ps:contact', {}), {
3607
- 'doc': 'Current profile details associated with the account.'}),
3649
+ ('tenant', ('inet:service:tenant', {}), {
3650
+ 'doc': 'The tenant which contains the account.'}),
3608
3651
  )),
3609
3652
 
3610
3653
  ('inet:service:relationship:type:taxonomy', {}, ()),
@@ -3621,10 +3664,7 @@ class InetModule(s_module.CoreModule):
3621
3664
  'doc': 'The type of relationship between the source and the target.'}),
3622
3665
  )),
3623
3666
 
3624
- ('inet:service:group', {}, ( # inet:service:object
3625
-
3626
- ('id', ('str', {'strip': True}), {
3627
- 'doc': 'A platform specific ID used to identify the group.'}),
3667
+ ('inet:service:group', {}, (
3628
3668
 
3629
3669
  ('name', ('inet:group', {}), {
3630
3670
  'doc': 'The name of the group on this platform.'}),
@@ -3674,14 +3714,14 @@ class InetModule(s_module.CoreModule):
3674
3714
 
3675
3715
  ('inet:service:session', {}, (
3676
3716
 
3677
- ('id', ('str', {'strip': True}), {
3678
- 'doc': 'The service specific session id.'}),
3679
-
3680
3717
  ('creator', ('inet:service:account', {}), {
3681
3718
  'doc': 'The account which authenticated to create the session.'}),
3682
3719
 
3683
3720
  ('period', ('ival', {}), {
3684
3721
  'doc': 'The period where the session was valid.'}),
3722
+
3723
+ ('http:session', ('inet:http:session', {}), {
3724
+ 'doc': 'The HTTP session associated with the service session.'}),
3685
3725
  )),
3686
3726
 
3687
3727
  ('inet:service:login', {}, (
@@ -3742,7 +3782,8 @@ class InetModule(s_module.CoreModule):
3742
3782
  'doc': 'The name of the place that the message was sent from.'}),
3743
3783
 
3744
3784
  ('client:address', ('inet:client', {}), {
3745
- 'doc': 'The client address that the message was sent from.'}),
3785
+ 'deprecated': True,
3786
+ 'doc': 'Deprecated. Please use :client'}),
3746
3787
 
3747
3788
  ('client:software', ('it:prod:softver', {}), {
3748
3789
  'doc': 'The client software version used to send the message.'}),
@@ -3864,6 +3905,22 @@ class InetModule(s_module.CoreModule):
3864
3905
  ('type', ('int', {'enums': svcaccesstypes}), {
3865
3906
  'doc': 'The type of access requested.'}),
3866
3907
  )),
3908
+
3909
+ ('inet:service:tenant', {}, ()),
3910
+
3911
+ ('inet:service:subscription:level:taxonomy', {}, ()),
3912
+
3913
+ ('inet:service:subscription', {}, (
3914
+
3915
+ ('level', ('inet:service:subscription:level:taxonomy', {}), {
3916
+ 'doc': 'A platform specific subscription level.'}),
3917
+
3918
+ ('pay:instrument', ('econ:pay:instrument', {}), {
3919
+ 'doc': 'The primary payment instrument used to pay for the subscription.'}),
3920
+
3921
+ ('subscriber', ('inet:service:subscriber', {}), {
3922
+ 'doc': 'The subscriber who owns the subscription.'}),
3923
+ )),
3867
3924
  ),
3868
3925
  }),
3869
3926
  )
synapse/models/person.py CHANGED
@@ -377,10 +377,12 @@ class PsModule(s_module.CoreModule):
377
377
  'doc': 'The service accounts associated with this contact.'}),
378
378
 
379
379
  ('web:acct', ('inet:web:acct', {}), {
380
- 'doc': 'The social media account for this contact.',
380
+ 'deprecated': True,
381
+ 'doc': 'Deprecated. Use :service:accounts.',
381
382
  }),
382
383
  ('web:group', ('inet:web:group', {}), {
383
- 'doc': 'A web group representing this contact.',
384
+ 'deprecated': True,
385
+ 'doc': 'Deprecated. Use inet:service:group:profile to link to a group.',
384
386
  }),
385
387
  ('birth:place', ('geo:place', {}), {
386
388
  'doc': 'A fully resolved place of birth for this contact.',
@@ -463,7 +465,8 @@ class PsModule(s_module.CoreModule):
463
465
  'doc': 'An array of secondary/associated email addresses.',
464
466
  }),
465
467
  ('web:accts', ('array', {'type': 'inet:web:acct', 'uniq': True, 'sorted': True}), {
466
- 'doc': 'An array of secondary/associated web accounts.',
468
+ 'deprecated': True,
469
+ 'doc': 'Deprecated. Use :service:accounts.',
467
470
  }),
468
471
  ('id:numbers', ('array', {'type': 'ou:id:number', 'uniq': True, 'sorted': True}), {
469
472
  'doc': 'An array of secondary/associated IDs.',
@@ -511,7 +514,11 @@ class PsModule(s_module.CoreModule):
511
514
  'doc': 'The file from which the contact list was extracted.',
512
515
  }),
513
516
  ('source:acct', ('inet:web:acct', {}), {
514
- 'doc': 'The web account from which the contact list was extracted.',
517
+ 'deprecated': True,
518
+ 'doc': 'Deprecated. Use :source:account.',
519
+ }),
520
+ ('source:account', ('inet:service:account', {}), {
521
+ 'doc': 'The service account from which the contact list was extracted.',
515
522
  }),
516
523
  )),
517
524
 
synapse/models/risk.py CHANGED
@@ -279,6 +279,9 @@ class RiskModule(s_module.CoreModule):
279
279
  ('active', ('ival', {}), {
280
280
  'doc': 'An interval for when the threat cluster is assessed to have been active.'}),
281
281
 
282
+ ('activity', ('meta:activity', {}), {
283
+ 'doc': 'The most recently assessed activity level of the threat cluster.'}),
284
+
282
285
  ('reporter', ('ou:org', {}), {
283
286
  'doc': 'The organization reporting on the threat cluster.'}),
284
287
 
@@ -1078,6 +1081,9 @@ class RiskModule(s_module.CoreModule):
1078
1081
  'ex': 'nature.earthquake',
1079
1082
  'doc': 'The outage cause type.'}),
1080
1083
 
1084
+ ('attack', ('risk:attack', {}), {
1085
+ 'doc': 'An attack which caused the outage.'}),
1086
+
1081
1087
  ('provider', ('ou:org', {}), {
1082
1088
  'doc': 'The organization which experienced the outage event.'}),
1083
1089
 
synapse/models/syn.py CHANGED
@@ -83,23 +83,28 @@ class SynModule(s_module.CoreModule):
83
83
 
84
84
  async def _liftRuntSynCmd(self, full, valu=None, cmpr=None, view=None):
85
85
 
86
- genr = self.core.stormcmds.values
86
+ def iterStormCmds():
87
+ for item in self.core.getStormCmds():
88
+ yield item[1]
87
89
 
88
- async for node in self._doRuntLift(genr, full, valu, cmpr):
90
+ async for node in self._doRuntLift(iterStormCmds, full, valu, cmpr):
89
91
  yield node
90
92
 
91
93
  async def _liftRuntSynCron(self, full, valu=None, cmpr=None, view=None):
92
94
 
93
- genr = self.core.agenda.appts.values
95
+ def iterAppts():
96
+ for item in self.core.agenda.list():
97
+ yield item[1]
94
98
 
95
- async for node in self._doRuntLift(genr, full, valu, cmpr):
99
+ async for node in self._doRuntLift(iterAppts, full, valu, cmpr):
96
100
  yield node
97
101
 
98
102
  async def _liftRuntSynForm(self, full, valu=None, cmpr=None, view=None):
99
103
 
100
- genr = self.model.forms.values
104
+ def getForms():
105
+ return list(self.model.forms.values())
101
106
 
102
- async for node in self._doRuntLift(genr, full, valu, cmpr):
107
+ async for node in self._doRuntLift(getForms, full, valu, cmpr):
103
108
  yield node
104
109
 
105
110
  async def _liftRuntSynProp(self, full, valu=None, cmpr=None, view=None):
@@ -111,24 +116,29 @@ class SynModule(s_module.CoreModule):
111
116
 
112
117
  async def _liftRuntSynType(self, full, valu=None, cmpr=None, view=None):
113
118
 
114
- genr = self.model.types.values
119
+ def getTypes():
120
+ return list(self.model.types.values())
115
121
 
116
- async for node in self._doRuntLift(genr, full, valu, cmpr):
122
+ async for node in self._doRuntLift(getTypes, full, valu, cmpr):
117
123
  yield node
118
124
 
119
125
  async def _liftRuntSynTagProp(self, full, valu=None, cmpr=None, view=None):
120
126
 
121
- genr = self.model.tagprops.values
127
+ def getTagProps():
128
+ return list(self.model.tagprops.values())
122
129
 
123
- async for node in self._doRuntLift(genr, full, valu, cmpr):
130
+ async for node in self._doRuntLift(getTagProps, full, valu, cmpr):
124
131
  yield node
125
132
 
126
133
  async def _liftRuntSynTrigger(self, full, valu=None, cmpr=None, view=None):
127
134
 
128
135
  view = self.core.getView(iden=view)
129
- genr = view.triggers.triggers.values
130
136
 
131
- async for node in self._doRuntLift(genr, full, valu, cmpr):
137
+ def iterTriggers():
138
+ for item in view.triggers.list():
139
+ yield item[1]
140
+
141
+ async for node in self._doRuntLift(iterTriggers, full, valu, cmpr):
132
142
  yield node
133
143
 
134
144
  async def _doRuntLift(self, genr, full, valu=None, cmpr=None):
synapse/models/telco.py CHANGED
@@ -376,7 +376,9 @@ class TelcoModule(s_module.CoreModule):
376
376
  # User related data
377
377
  ('name', ('ps:name', {}), {}),
378
378
  ('email', ('inet:email', {}), {}),
379
- ('acct', ('inet:web:acct', {}), {}),
379
+ ('acct', ('inet:web:acct', {}), {
380
+ 'doc': 'Deprecated, use :account.',
381
+ 'deprecated': True}),
380
382
 
381
383
  ('account', ('inet:service:account', {}), {
382
384
  'doc': 'The service account which is associated with the tracked device.'}),
@@ -1,11 +1,9 @@
1
1
  import os
2
2
  import copy
3
- import json
4
3
  import time
5
4
  import asyncio
6
5
  import hashlib
7
6
  import logging
8
- import textwrap
9
7
 
10
8
  import regex
11
9
 
@@ -16,13 +14,13 @@ import synapse.common as s_common
16
14
  import synapse.cortex as s_cortex
17
15
  import synapse.telepath as s_telepath
18
16
 
19
- import synapse.lib.aha as s_aha
20
17
  import synapse.lib.base as s_base
21
18
  import synapse.lib.cell as s_cell
22
19
  import synapse.lib.coro as s_coro
23
20
  import synapse.lib.node as s_node
24
21
  import synapse.lib.time as s_time
25
22
  import synapse.lib.layer as s_layer
23
+ import synapse.lib.storm as s_storm
26
24
  import synapse.lib.output as s_output
27
25
  import synapse.lib.msgpack as s_msgpack
28
26
  import synapse.lib.version as s_version
@@ -1114,8 +1112,7 @@ class CortexTest(s_t_utils.SynTest):
1114
1112
  ''')
1115
1113
  self.true(await stream.wait(6))
1116
1114
 
1117
- buf = stream.getvalue()
1118
- mesg = json.loads(buf.split('\n')[0])
1115
+ mesg = stream.jsonlines()[0]
1119
1116
  self.eq(mesg.get('message'), f'Running dmon {iden}')
1120
1117
  self.eq(mesg.get('iden'), iden)
1121
1118
 
@@ -3365,8 +3362,7 @@ class CortexBasicTest(s_t_utils.SynTest):
3365
3362
  await alist(core.storm('help foo', opts={'show': ('init', 'fini', 'print',)}))
3366
3363
  self.true(await stream.wait(4))
3367
3364
 
3368
- buf = stream.getvalue()
3369
- mesg = json.loads(buf.split('\n')[0])
3365
+ mesg = stream.jsonlines()[0]
3370
3366
  self.eq(mesg.get('view'), view)
3371
3367
 
3372
3368
  async def test_strict(self):
@@ -7901,8 +7897,7 @@ class CortexBasicTest(s_t_utils.SynTest):
7901
7897
  self.eq('admin', whoami)
7902
7898
  self.eq('lowuser', udef.get('name'))
7903
7899
 
7904
- raw_mesgs = [m for m in stream.getvalue().split('\n') if m]
7905
- msgs = [json.loads(m) for m in raw_mesgs]
7900
+ msgs = stream.jsonlines()
7906
7901
  mesg = [m for m in msgs if 'Added user' in m.get('message')][0]
7907
7902
  self.eq('Added user=lowuser', mesg.get('message'))
7908
7903
  self.eq('admin', mesg.get('username'))
@@ -7916,8 +7911,7 @@ class CortexBasicTest(s_t_utils.SynTest):
7916
7911
  msgs.append(mesg)
7917
7912
  self.stormHasNoWarnErr(msgs)
7918
7913
 
7919
- raw_mesgs = [m for m in stream.getvalue().split('\n') if m]
7920
- msgs = [json.loads(m) for m in raw_mesgs]
7914
+ msgs = stream.jsonlines()
7921
7915
  mesg = [m for m in msgs if 'Set admin' in m.get('message')][0]
7922
7916
  self.isin('Set admin=True for lowuser', mesg.get('message'))
7923
7917
  self.eq('admin', mesg.get('username'))
@@ -8072,6 +8066,8 @@ class CortexBasicTest(s_t_utils.SynTest):
8072
8066
 
8073
8067
  async with self.getTestAha() as aha:
8074
8068
 
8069
+ ahanet = aha.conf.req('aha:network')
8070
+
8075
8071
  async with await s_base.Base.anit() as base:
8076
8072
 
8077
8073
  with self.getTestDir() as dirn:
@@ -8107,43 +8103,89 @@ class CortexBasicTest(s_t_utils.SynTest):
8107
8103
  self.stormIsInPrint('Storm pool configuration set.', msgs)
8108
8104
 
8109
8105
  await core00.fini()
8106
+ await core01.fini()
8110
8107
 
8111
8108
  core00 = await base.enter_context(self.getTestCore(dirn=dirn00))
8109
+ core01 = await base.enter_context(self.getTestCore(dirn=dirn01, conf={'storm:log': True}))
8112
8110
 
8113
8111
  await core00.stormpool.waitready(timeout=12)
8114
8112
 
8115
- with self.getLoggerStream('synapse') as stream:
8116
- msgs = await alist(core00.storm('inet:asn=0'))
8113
+ # storm()
8114
+ q = 'inet:asn=0'
8115
+ qhash = s_storm.queryhash(q)
8116
+ with self.getStructuredAsyncLoggerStream('synapse') as stream:
8117
+ msgs = await alist(core00.storm(q))
8117
8118
  self.len(1, [m for m in msgs if m[0] == 'node'])
8118
8119
 
8119
- stream.seek(0)
8120
- data = stream.read()
8121
- self.isin('Offloading Storm query', data)
8120
+ data = stream.getvalue()
8122
8121
  self.notin('Timeout', data)
8122
+ msgs = stream.jsonlines()
8123
+ self.len(2, msgs)
8123
8124
 
8124
- with self.getLoggerStream('synapse') as stream:
8125
- self.true(await core00.callStorm('inet:asn=0 return($lib.true)'))
8125
+ self.eq(msgs[0].get('message'), f'Offloading Storm query to mirror 01.core.{ahanet}.')
8126
+ self.eq(msgs[0].get('hash'), qhash)
8127
+ self.eq(msgs[0].get('mirror'), f'01.core.{ahanet}')
8126
8128
 
8127
- stream.seek(0)
8128
- data = stream.read()
8129
- self.isin('Offloading Storm query', data)
8129
+ self.eq(msgs[1].get('message'), f'Executing storm query {{{q}}} as [root]')
8130
+ self.eq(msgs[1].get('hash'), qhash)
8131
+ self.eq(msgs[1].get('pool:from'), f'00.core.{ahanet}')
8132
+
8133
+ # callStorm()
8134
+ q = 'inet:asn=0 return($lib.true)'
8135
+ qhash = s_storm.queryhash(q)
8136
+ with self.getStructuredAsyncLoggerStream('synapse') as stream:
8137
+ self.true(await core00.callStorm(q))
8138
+
8139
+ data = stream.getvalue()
8130
8140
  self.notin('Timeout', data)
8141
+ msgs = stream.jsonlines()
8142
+ self.len(2, msgs)
8131
8143
 
8132
- with self.getLoggerStream('synapse') as stream:
8133
- self.len(1, await alist(core00.exportStorm('inet:asn=0')))
8144
+ self.eq(msgs[0].get('message'), f'Offloading Storm query to mirror 01.core.{ahanet}.')
8145
+ self.eq(msgs[0].get('hash'), qhash)
8146
+ self.eq(msgs[0].get('mirror'), f'01.core.{ahanet}')
8134
8147
 
8135
- stream.seek(0)
8136
- data = stream.read()
8137
- self.isin('Offloading Storm query', data)
8148
+ self.eq(msgs[1].get('message'), f'Executing storm query {{{q}}} as [root]')
8149
+ self.eq(msgs[1].get('hash'), qhash)
8150
+ self.eq(msgs[1].get('pool:from'), f'00.core.{ahanet}')
8151
+
8152
+ # exportStorm()
8153
+ q = 'inet:asn=0'
8154
+ qhash = s_storm.queryhash(q)
8155
+ with self.getStructuredAsyncLoggerStream('synapse') as stream:
8156
+ self.len(1, await alist(core00.exportStorm(q)))
8157
+
8158
+ data = stream.getvalue()
8138
8159
  self.notin('Timeout', data)
8160
+ msgs = stream.jsonlines()
8161
+ self.len(2, msgs)
8139
8162
 
8140
- with self.getLoggerStream('synapse') as stream:
8141
- self.eq(1, await core00.count('inet:asn=0'))
8163
+ self.eq(msgs[0].get('message'), f'Offloading Storm query to mirror 01.core.{ahanet}.')
8164
+ self.eq(msgs[0].get('hash'), qhash)
8165
+ self.eq(msgs[0].get('mirror'), f'01.core.{ahanet}')
8142
8166
 
8143
- stream.seek(0)
8144
- data = stream.read()
8145
- self.isin('Offloading Storm query', data)
8167
+ self.eq(msgs[1].get('message'), f'Executing storm query {{{q}}} as [root]')
8168
+ self.eq(msgs[1].get('hash'), qhash)
8169
+ self.eq(msgs[1].get('pool:from'), f'00.core.{ahanet}')
8170
+
8171
+ # count()
8172
+ q = 'inet:asn=0'
8173
+ qhash = s_storm.queryhash(q)
8174
+ with self.getStructuredAsyncLoggerStream('synapse') as stream:
8175
+ self.eq(1, await core00.count(q))
8176
+
8177
+ data = stream.getvalue()
8146
8178
  self.notin('Timeout', data)
8179
+ msgs = stream.jsonlines()
8180
+ self.len(2, msgs)
8181
+
8182
+ self.eq(msgs[0].get('message'), f'Offloading Storm query to mirror 01.core.{ahanet}.')
8183
+ self.eq(msgs[0].get('hash'), qhash)
8184
+ self.eq(msgs[0].get('mirror'), f'01.core.{ahanet}')
8185
+
8186
+ self.eq(msgs[1].get('message'), f'Executing storm query {{{q}}} as [root]')
8187
+ self.eq(msgs[1].get('hash'), qhash)
8188
+ self.eq(msgs[1].get('pool:from'), f'00.core.{ahanet}')
8147
8189
 
8148
8190
  with patch('synapse.cortex.CoreApi.getNexsIndx', _hang):
8149
8191
 
@@ -1,4 +1,3 @@
1
- import json
2
1
  import asyncio
3
2
  import hashlib
4
3
  import datetime
@@ -9,8 +8,6 @@ import synapse.exc as s_exc
9
8
  import synapse.common as s_common
10
9
  import synapse.tests.utils as s_t_utils
11
10
 
12
- import synapse.lib.hive as s_hive
13
- import synapse.lib.lmdbslab as s_lmdbslab
14
11
  import synapse.tools.backup as s_tools_backup
15
12
 
16
13
  import synapse.lib.agenda as s_agenda
@@ -401,9 +398,7 @@ class AgendaTest(s_t_utils.SynTest):
401
398
  self.eq((12, 'bar'), await asyncio.wait_for(core.callStorm('return($lib.queue.gen(visi).pop(wait=$lib.true))'), timeout=5))
402
399
  core.stormlog = False
403
400
 
404
- data = stream.getvalue()
405
- raw_mesgs = [m for m in data.split('\n') if m]
406
- msgs = [json.loads(m) for m in raw_mesgs]
401
+ msgs = stream.jsonlines()
407
402
  msgs = [m for m in msgs if m['text'] == '$lib.queue.gen(visi).put(bar)']
408
403
  self.gt(len(msgs), 0)
409
404
  for m in msgs: