synapse 2.176.0__py311-none-any.whl → 2.177.0__py311-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of synapse might be problematic. Click here for more details.
- synapse/axon.py +24 -9
- synapse/cortex.py +329 -168
- synapse/cryotank.py +46 -37
- synapse/datamodel.py +17 -4
- synapse/exc.py +19 -0
- synapse/lib/agenda.py +7 -13
- synapse/lib/auth.py +1520 -0
- synapse/lib/cell.py +255 -53
- synapse/lib/grammar.py +5 -0
- synapse/lib/hive.py +24 -3
- synapse/lib/hiveauth.py +6 -32
- synapse/lib/layer.py +7 -4
- synapse/lib/link.py +21 -17
- synapse/lib/lmdbslab.py +149 -0
- synapse/lib/modelrev.py +1 -1
- synapse/lib/schemas.py +136 -0
- synapse/lib/storm.py +61 -29
- synapse/lib/stormlib/aha.py +1 -1
- synapse/lib/stormlib/auth.py +185 -10
- synapse/lib/stormlib/cortex.py +16 -5
- synapse/lib/stormlib/gen.py +80 -0
- synapse/lib/stormlib/model.py +55 -0
- synapse/lib/stormlib/modelext.py +60 -0
- synapse/lib/stormlib/tabular.py +212 -0
- synapse/lib/stormtypes.py +14 -1
- synapse/lib/trigger.py +1 -1
- synapse/lib/version.py +2 -2
- synapse/lib/view.py +55 -28
- synapse/models/base.py +7 -0
- synapse/models/biz.py +4 -0
- synapse/models/files.py +8 -1
- synapse/models/inet.py +8 -0
- synapse/tests/files/changelog/model_2.176.0_16ee721a6b7221344eaf946c3ab4602dda546b1a.yaml.gz +0 -0
- synapse/tests/files/changelog/model_2.176.0_2a25c58bbd344716cd7cbc3f4304d8925b0f4ef2.yaml.gz +0 -0
- synapse/tests/test_axon.py +7 -4
- synapse/tests/test_cortex.py +127 -82
- synapse/tests/test_cryotank.py +4 -4
- synapse/tests/test_datamodel.py +7 -0
- synapse/tests/test_lib_agenda.py +7 -0
- synapse/tests/{test_lib_hiveauth.py → test_lib_auth.py} +314 -11
- synapse/tests/test_lib_cell.py +161 -8
- synapse/tests/test_lib_httpapi.py +18 -14
- synapse/tests/test_lib_layer.py +33 -33
- synapse/tests/test_lib_link.py +42 -1
- synapse/tests/test_lib_lmdbslab.py +68 -0
- synapse/tests/test_lib_nexus.py +4 -4
- synapse/tests/test_lib_node.py +0 -7
- synapse/tests/test_lib_storm.py +45 -0
- synapse/tests/test_lib_stormlib_aha.py +1 -2
- synapse/tests/test_lib_stormlib_auth.py +21 -0
- synapse/tests/test_lib_stormlib_cortex.py +12 -12
- synapse/tests/test_lib_stormlib_gen.py +99 -0
- synapse/tests/test_lib_stormlib_model.py +108 -0
- synapse/tests/test_lib_stormlib_modelext.py +64 -0
- synapse/tests/test_lib_stormlib_tabular.py +226 -0
- synapse/tests/test_lib_stormsvc.py +4 -1
- synapse/tests/test_lib_stormtypes.py +10 -0
- synapse/tests/test_model_base.py +3 -0
- synapse/tests/test_model_biz.py +3 -0
- synapse/tests/test_model_files.py +12 -2
- synapse/tests/test_model_inet.py +24 -0
- synapse/tests/test_tools_changelog.py +196 -0
- synapse/tests/test_tools_healthcheck.py +4 -3
- synapse/tests/utils.py +1 -1
- synapse/tools/changelog.py +774 -15
- {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/METADATA +3 -3
- {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/RECORD +70 -64
- {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/WHEEL +1 -1
- {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/LICENSE +0 -0
- {synapse-2.176.0.dist-info → synapse-2.177.0.dist-info}/top_level.txt +0 -0
synapse/lib/stormlib/model.py
CHANGED
|
@@ -919,8 +919,63 @@ class LibModelMigrations(s_stormtypes.Lib, MigrationEditorMixin):
|
|
|
919
919
|
'desc': 'Do not copy nodedata to the risk:vulnerable node.'},
|
|
920
920
|
),
|
|
921
921
|
'returns': {'type': 'list', 'desc': 'A list of idens for the risk:vulnerable nodes.'}}},
|
|
922
|
+
{'name': 'inetSslCertToTlsServerCert', 'desc': '''
|
|
923
|
+
Create a inet:tls:servercert node from the provided inet:ssl:cert node.
|
|
924
|
+
|
|
925
|
+
Edits will be made to the inet:tls:servercert node in the current write layer.
|
|
926
|
+
|
|
927
|
+
Tags, tag properties, edges, and node data will be copied
|
|
928
|
+
to the inet:tls:servercert node. However, existing tag properties and
|
|
929
|
+
node data will not be overwritten.
|
|
930
|
+
''',
|
|
931
|
+
'type': {'type': 'function', '_funcname': '_storm_query',
|
|
932
|
+
'args': (
|
|
933
|
+
{'name': 'n', 'type': 'node', 'desc': 'The inet:ssl:cert node to migrate.'},
|
|
934
|
+
{'name': 'nodata', 'type': 'bool', 'default': False,
|
|
935
|
+
'desc': 'Do not copy nodedata to the inet:tls:servercert node.'},
|
|
936
|
+
),
|
|
937
|
+
'returns': {'type': 'node', 'desc': 'The newly created inet:tls:servercert node.'}}},
|
|
938
|
+
|
|
922
939
|
)
|
|
923
940
|
_storm_lib_path = ('model', 'migration', 's')
|
|
941
|
+
_storm_query = '''
|
|
942
|
+
function inetSslCertToTlsServerCert(n, nodata=$lib.false) {
|
|
943
|
+
$form = $n.form()
|
|
944
|
+
if ($form != 'inet:ssl:cert') {
|
|
945
|
+
$mesg = `$lib.model.migration.s.inetSslCertToTlsServerCert() only accepts inet:ssl:cert nodes, not {$form}`
|
|
946
|
+
$lib.raise(BadArg, $mesg)
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
$server = $n.props.server
|
|
950
|
+
$sha256 = { yield $n -> file:bytes -> hash:sha256 }
|
|
951
|
+
|
|
952
|
+
if $sha256 {
|
|
953
|
+
|
|
954
|
+
yield $lib.gen.inetTlsServerCertByServerAndSha256($server, $sha256)
|
|
955
|
+
|
|
956
|
+
} else {
|
|
957
|
+
|
|
958
|
+
// File doesn't have a :sha256, try to lift/create a crypto:x509:node based on the file link
|
|
959
|
+
$crypto = { yield $n -> file:bytes -> crypto:x509:cert:file }
|
|
960
|
+
if (not $crypto) {
|
|
961
|
+
$crypto = {[ crypto:x509:cert=($n.props.file,) :file=$n.props.file ]}
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
[ inet:tls:servercert=($server, $crypto) ]
|
|
965
|
+
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
[ .seen ?= $n.props.".seen" ]
|
|
969
|
+
|
|
970
|
+
$lib.model.migration.copyTags($n, $node, overwrite=$lib.false)
|
|
971
|
+
$lib.model.migration.copyEdges($n, $node)
|
|
972
|
+
if (not $nodata) {
|
|
973
|
+
$lib.model.migration.copyData($n, $node, overwrite=$lib.false)
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
return($node)
|
|
977
|
+
}
|
|
978
|
+
'''
|
|
924
979
|
|
|
925
980
|
def getObjLocals(self):
|
|
926
981
|
return {
|
synapse/lib/stormlib/modelext.py
CHANGED
|
@@ -77,6 +77,27 @@ class LibModelExt(s_stormtypes.Lib):
|
|
|
77
77
|
{'name': 'model', 'type': 'dict', 'desc': 'A model dictionary from getExtModel().', },
|
|
78
78
|
),
|
|
79
79
|
'returns': {'type': 'boolean'}}},
|
|
80
|
+
{'name': 'addEdge', 'desc': 'Add an extended edge definition to the data model.',
|
|
81
|
+
'type': {'type': 'function', '_funcname': 'addEdge',
|
|
82
|
+
'args': (
|
|
83
|
+
{'name': 'n1form', 'type': 'str',
|
|
84
|
+
'desc': 'The form of the n1 node. May be "*" or null to specify "any".'},
|
|
85
|
+
{'name': 'verb', 'type': 'str', 'desc': 'The edge verb, which must begin with "_".'},
|
|
86
|
+
{'name': 'n2form', 'type': 'str',
|
|
87
|
+
'desc': 'The form of the n2 node. May be "*" or null to specify "any".'},
|
|
88
|
+
{'name': 'edgeinfo', 'type': 'dict', 'desc': 'A Synapse edge info dictionary.'},
|
|
89
|
+
),
|
|
90
|
+
'returns': {'type': 'null'}}},
|
|
91
|
+
{'name': 'delEdge', 'desc': 'Remove an extended edge definition from the data model.',
|
|
92
|
+
'type': {'type': 'function', '_funcname': 'delEdge',
|
|
93
|
+
'args': (
|
|
94
|
+
{'name': 'n1form', 'type': 'str',
|
|
95
|
+
'desc': 'The form of the n1 node. May be "*" or null to specify "any".'},
|
|
96
|
+
{'name': 'verb', 'type': 'str', 'desc': 'The edge verb, which must begin with "_".'},
|
|
97
|
+
{'name': 'n2form', 'type': 'str',
|
|
98
|
+
'desc': 'The form of the n2 node. May be "*" or null to specify "any".'},
|
|
99
|
+
),
|
|
100
|
+
'returns': {'type': 'null'}}},
|
|
80
101
|
)
|
|
81
102
|
_storm_lib_path = ('model', 'ext')
|
|
82
103
|
|
|
@@ -92,6 +113,8 @@ class LibModelExt(s_stormtypes.Lib):
|
|
|
92
113
|
'delTagProp': self.delTagProp,
|
|
93
114
|
'getExtModel': self.getExtModel,
|
|
94
115
|
'addExtModel': self.addExtModel,
|
|
116
|
+
'addEdge': self.addEdge,
|
|
117
|
+
'delEdge': self.delEdge,
|
|
95
118
|
}
|
|
96
119
|
|
|
97
120
|
# TODO type docs in the new convention
|
|
@@ -163,3 +186,40 @@ class LibModelExt(s_stormtypes.Lib):
|
|
|
163
186
|
async def addExtModel(self, model):
|
|
164
187
|
model = await s_stormtypes.toprim(model)
|
|
165
188
|
return await self.runt.snap.core.addExtModel(model)
|
|
189
|
+
|
|
190
|
+
async def addEdge(self, n1form, verb, n2form, edgeinfo):
|
|
191
|
+
verb = await s_stormtypes.tostr(verb)
|
|
192
|
+
n1form = await s_stormtypes.tostr(n1form, noneok=True)
|
|
193
|
+
n2form = await s_stormtypes.tostr(n2form, noneok=True)
|
|
194
|
+
edgeinfo = await s_stormtypes.toprim(edgeinfo)
|
|
195
|
+
|
|
196
|
+
if not (s_grammar.isEdgeVerb(verb) and verb.islower()):
|
|
197
|
+
mesg = f'Invalid edge verb {verb}'
|
|
198
|
+
raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
|
|
199
|
+
|
|
200
|
+
if n1form == '*':
|
|
201
|
+
n1form = None
|
|
202
|
+
|
|
203
|
+
if n2form == '*':
|
|
204
|
+
n2form = None
|
|
205
|
+
|
|
206
|
+
s_stormtypes.confirm(('model', 'edge', 'add'))
|
|
207
|
+
await self.runt.snap.core.addEdge((n1form, verb, n2form), edgeinfo)
|
|
208
|
+
|
|
209
|
+
async def delEdge(self, n1form, verb, n2form):
|
|
210
|
+
verb = await s_stormtypes.tostr(verb)
|
|
211
|
+
n1form = await s_stormtypes.tostr(n1form, noneok=True)
|
|
212
|
+
n2form = await s_stormtypes.tostr(n2form, noneok=True)
|
|
213
|
+
|
|
214
|
+
if not (s_grammar.isEdgeVerb(verb) and verb.islower()):
|
|
215
|
+
mesg = f'Invalid edge verb {verb}'
|
|
216
|
+
raise s_exc.BadEdgeDef(mesg=mesg, n1form=n1form, verb=verb, n2form=n2form)
|
|
217
|
+
|
|
218
|
+
if n1form == '*':
|
|
219
|
+
n1form = None
|
|
220
|
+
|
|
221
|
+
if n2form == '*':
|
|
222
|
+
n2form = None
|
|
223
|
+
|
|
224
|
+
s_stormtypes.confirm(('model', 'edge', 'del'))
|
|
225
|
+
await self.runt.snap.core.delEdge((n1form, verb, n2form))
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import textwrap
|
|
3
|
+
import itertools
|
|
4
|
+
|
|
5
|
+
import synapse.exc as s_exc
|
|
6
|
+
import synapse.common as s_common
|
|
7
|
+
import synapse.lib.schemas as s_schemas
|
|
8
|
+
import synapse.lib.stormtypes as s_stormtypes
|
|
9
|
+
|
|
10
|
+
justers = {
|
|
11
|
+
'left': str.ljust,
|
|
12
|
+
'center': str.center,
|
|
13
|
+
'right': str.rjust,
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
@s_stormtypes.registry.registerLib
|
|
17
|
+
class LibTabular(s_stormtypes.Lib):
|
|
18
|
+
'''
|
|
19
|
+
A Storm Library for creating printable tables.
|
|
20
|
+
'''
|
|
21
|
+
_storm_locals = (
|
|
22
|
+
{'name': 'printer', 'desc': '''
|
|
23
|
+
Construct a new printer.
|
|
24
|
+
|
|
25
|
+
Examples:
|
|
26
|
+
Create a simple table using the default separators::
|
|
27
|
+
|
|
28
|
+
$conf = ({
|
|
29
|
+
"columns": [
|
|
30
|
+
{"name": "Year", "width": 4},
|
|
31
|
+
{"name": "Author", "width": 20},
|
|
32
|
+
{"name": "Title", "width": 12},
|
|
33
|
+
]
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
$printer = $lib.tabular.printer($conf)
|
|
37
|
+
|
|
38
|
+
$lib.print($printer.header())
|
|
39
|
+
|
|
40
|
+
for ($year, $author, $title, $publisher) in $data {
|
|
41
|
+
$lib.print($printer.row(($year, $author, $title))
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
Create a configuration with custom separators and column options::
|
|
45
|
+
|
|
46
|
+
$conf = ({
|
|
47
|
+
"separators": {
|
|
48
|
+
"row:outline": true,
|
|
49
|
+
"column:outline": true,
|
|
50
|
+
"header:row": "#",
|
|
51
|
+
"data:row": "*",
|
|
52
|
+
"column": "+",
|
|
53
|
+
},
|
|
54
|
+
"columns": [
|
|
55
|
+
{"name": "Year", "width": 4, "justify": "right"},
|
|
56
|
+
{"name": "Author", "width": 20, "justify": "center"},
|
|
57
|
+
{"name": "Title", "width": 12, "overflow": "wrap"},
|
|
58
|
+
]
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
$printer = $lib.tabular.printer($conf)
|
|
62
|
+
''',
|
|
63
|
+
'type': {'type': 'function', '_funcname': '_methPrinter',
|
|
64
|
+
'args': (
|
|
65
|
+
{'name': 'conf', 'type': 'dict',
|
|
66
|
+
'desc': 'The table configuration dictionary.'},
|
|
67
|
+
),
|
|
68
|
+
'returns': {'type': 'tabular:printer',
|
|
69
|
+
'desc': 'The newly constructed tabular:printer.'}}},
|
|
70
|
+
{'name': 'schema', 'desc': '''
|
|
71
|
+
Get a copy of the table configuration schema.
|
|
72
|
+
|
|
73
|
+
Examples:
|
|
74
|
+
Print a human-readable version of the schema::
|
|
75
|
+
|
|
76
|
+
$schema = $lib.tabular.schema()
|
|
77
|
+
$lib.print($lib.yaml.save($schema))
|
|
78
|
+
''',
|
|
79
|
+
'type': {'type': 'function', '_funcname': '_methSchema',
|
|
80
|
+
'returns': {'type': 'dict',
|
|
81
|
+
'desc': 'The table configuration schema.'}}},
|
|
82
|
+
)
|
|
83
|
+
_storm_lib_path = ('tabular',)
|
|
84
|
+
|
|
85
|
+
def getObjLocals(self):
|
|
86
|
+
return {
|
|
87
|
+
'printer': self._methPrinter,
|
|
88
|
+
'schema': self._methSchema,
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async def _methSchema(self):
|
|
92
|
+
return copy.deepcopy(s_schemas.tabularConfSchema)
|
|
93
|
+
|
|
94
|
+
async def _methPrinter(self, conf):
|
|
95
|
+
conf = await s_stormtypes.toprim(conf)
|
|
96
|
+
conf.setdefault('separators', {})
|
|
97
|
+
conf = s_schemas.reqValidTabularConf(conf)
|
|
98
|
+
return TabularPrinter(self.runt, conf)
|
|
99
|
+
|
|
100
|
+
@s_stormtypes.registry.registerType
|
|
101
|
+
class TabularPrinter(s_stormtypes.StormType):
|
|
102
|
+
'''
|
|
103
|
+
A Storm object for printing tabular data using a defined configuration.
|
|
104
|
+
'''
|
|
105
|
+
_storm_typename = 'tabular:printer'
|
|
106
|
+
_storm_locals = (
|
|
107
|
+
{'name': 'row', 'desc': 'Create a new row string from a data list.',
|
|
108
|
+
'type': {'type': 'function', '_funcname': 'row',
|
|
109
|
+
'args': (
|
|
110
|
+
{'name': 'data', 'type': 'list',
|
|
111
|
+
'desc': 'The data to create the row from; length must match the number of configured columns.'},
|
|
112
|
+
),
|
|
113
|
+
'returns': {'type': 'str', 'desc': 'The row string.'}}},
|
|
114
|
+
{'name': 'header',
|
|
115
|
+
'desc': 'Create a header row string.',
|
|
116
|
+
'type': {'type': 'function', '_funcname': 'header',
|
|
117
|
+
'returns': {'type': 'str', 'desc': 'The header row string.'}}},
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
def __init__(self, runt, conf):
|
|
121
|
+
s_stormtypes.StormType.__init__(self, None)
|
|
122
|
+
self.runt = runt
|
|
123
|
+
self.conf = conf
|
|
124
|
+
|
|
125
|
+
self.firstrow = True
|
|
126
|
+
|
|
127
|
+
self.seprconf = conf['separators']
|
|
128
|
+
|
|
129
|
+
self.colconf = conf['columns']
|
|
130
|
+
self.colcnt = len(self.colconf)
|
|
131
|
+
self.colwidths = [coldef.get('width') or len(coldef['name']) for coldef in self.colconf]
|
|
132
|
+
|
|
133
|
+
self.locls.update({
|
|
134
|
+
'row': self.row,
|
|
135
|
+
'header': self.header,
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
def _formatRowLine(self, lineitems, pad=' '):
|
|
139
|
+
sepr = self.seprconf['column']
|
|
140
|
+
endstr = sepr if self.seprconf['column:outline'] else ''
|
|
141
|
+
return f'{endstr}{pad}{f"{pad}{sepr}{pad}".join(lineitems)}{pad}{endstr}'
|
|
142
|
+
|
|
143
|
+
def _makeSeparatorRowStr(self, rowsepr):
|
|
144
|
+
if rowsepr:
|
|
145
|
+
lineitems = (width * rowsepr for width in self.colwidths)
|
|
146
|
+
return self._formatRowLine(lineitems, pad=rowsepr)
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
def _makeDataRowStrs(self, values):
|
|
150
|
+
|
|
151
|
+
items = []
|
|
152
|
+
for coldef, valu in zip(self.colconf, values):
|
|
153
|
+
width = coldef.get('width')
|
|
154
|
+
valu = str(valu) if valu is not None else ''
|
|
155
|
+
|
|
156
|
+
if width is None:
|
|
157
|
+
if coldef['newlines'] == 'split':
|
|
158
|
+
items.append(valu.split('\n'))
|
|
159
|
+
else:
|
|
160
|
+
items.append([valu.replace('\n', ' ')])
|
|
161
|
+
continue
|
|
162
|
+
|
|
163
|
+
valu = valu.replace('\n', ' ')
|
|
164
|
+
|
|
165
|
+
if len(valu) <= width:
|
|
166
|
+
items.append([justers[coldef['justify']](valu, width)])
|
|
167
|
+
continue
|
|
168
|
+
|
|
169
|
+
if coldef['overflow'] == 'trim':
|
|
170
|
+
items.append([s_common.trimText(valu, n=width)])
|
|
171
|
+
continue
|
|
172
|
+
|
|
173
|
+
items.append([justers[coldef['justify']](item, width) for item in textwrap.wrap(valu, width)])
|
|
174
|
+
|
|
175
|
+
rowstrs = []
|
|
176
|
+
for lineitems in itertools.zip_longest(*items, fillvalue=None):
|
|
177
|
+
lineitems = ([item or self.colwidths[i] * ' ' for i, item in enumerate(lineitems)])
|
|
178
|
+
rowstrs.append(self._formatRowLine(lineitems))
|
|
179
|
+
|
|
180
|
+
return rowstrs
|
|
181
|
+
|
|
182
|
+
async def header(self):
|
|
183
|
+
rowstrs = self._makeDataRowStrs([col['name'] for col in self.colconf])
|
|
184
|
+
|
|
185
|
+
if seprstr := self._makeSeparatorRowStr(self.seprconf['header:row']):
|
|
186
|
+
rowstrs.append(seprstr)
|
|
187
|
+
if self.seprconf['row:outline']:
|
|
188
|
+
rowstrs.insert(0, seprstr)
|
|
189
|
+
|
|
190
|
+
return '\n'.join(rowstrs)
|
|
191
|
+
|
|
192
|
+
async def row(self, data):
|
|
193
|
+
data = await s_stormtypes.toprim(data)
|
|
194
|
+
|
|
195
|
+
if not isinstance(data, (list, tuple)):
|
|
196
|
+
raise s_exc.BadArg(mesg='tabular:printer row() requires a list argument')
|
|
197
|
+
|
|
198
|
+
if len(data) != self.colcnt:
|
|
199
|
+
mesg = 'tabular:printer row() requires data length to equal column count'
|
|
200
|
+
raise s_exc.BadArg(mesg=mesg, length=len(data), column_length=self.colcnt)
|
|
201
|
+
|
|
202
|
+
rowstrs = self._makeDataRowStrs(data)
|
|
203
|
+
|
|
204
|
+
if seprstr := self._makeSeparatorRowStr(self.seprconf['data:row']):
|
|
205
|
+
if self.seprconf['row:outline']:
|
|
206
|
+
rowstrs.append(seprstr)
|
|
207
|
+
elif self.firstrow:
|
|
208
|
+
self.firstrow = False
|
|
209
|
+
else:
|
|
210
|
+
rowstrs.insert(0, seprstr)
|
|
211
|
+
|
|
212
|
+
return '\n'.join(rowstrs)
|
synapse/lib/stormtypes.py
CHANGED
|
@@ -4138,7 +4138,7 @@ class LibBase64(Lib):
|
|
|
4138
4138
|
if urlsafe:
|
|
4139
4139
|
return base64.urlsafe_b64decode(valu)
|
|
4140
4140
|
return base64.b64decode(valu)
|
|
4141
|
-
except binascii.Error as e:
|
|
4141
|
+
except (binascii.Error, TypeError) as e:
|
|
4142
4142
|
mesg = f'Error during base64 decoding - {str(e)}: {s_common.trimText(repr(valu))}'
|
|
4143
4143
|
raise s_exc.StormRuntimeError(mesg=mesg, urlsafe=urlsafe) from None
|
|
4144
4144
|
|
|
@@ -7532,6 +7532,9 @@ class View(Prim):
|
|
|
7532
7532
|
{'name': 'wipeLayer', 'desc': 'Delete all nodes and nodedata from the write layer. Triggers will be run.',
|
|
7533
7533
|
'type': {'type': 'function', '_funcname': '_methWipeLayer',
|
|
7534
7534
|
'returns': {'type': 'null', }}},
|
|
7535
|
+
{'name': 'swapLayer', 'desc': 'Swaps the top layer for a fresh one and deletes the old layer.',
|
|
7536
|
+
'type': {'type': 'function', '_funcname': '_methSwapLayer',
|
|
7537
|
+
'returns': {'type': 'null', }}},
|
|
7535
7538
|
{'name': 'addNode', 'desc': '''Transactionally add a single node and all it's properties. If any validation fails, no changes are made.''',
|
|
7536
7539
|
'type': {'type': 'function', '_funcname': 'addNode',
|
|
7537
7540
|
'args': (
|
|
@@ -7717,6 +7720,7 @@ class View(Prim):
|
|
|
7717
7720
|
'addNode': self.addNode,
|
|
7718
7721
|
'getEdges': self._methGetEdges,
|
|
7719
7722
|
'wipeLayer': self._methWipeLayer,
|
|
7723
|
+
'swapLayer': self._methSwapLayer,
|
|
7720
7724
|
'addNodeEdits': self._methAddNodeEdits,
|
|
7721
7725
|
'getEdgeVerbs': self._methGetEdgeVerbs,
|
|
7722
7726
|
'getFormCounts': self._methGetFormcount,
|
|
@@ -8017,6 +8021,15 @@ class View(Prim):
|
|
|
8017
8021
|
view = self.runt.snap.core.getView(viewiden)
|
|
8018
8022
|
await view.wipeLayer(useriden=useriden)
|
|
8019
8023
|
|
|
8024
|
+
async def _methSwapLayer(self):
|
|
8025
|
+
|
|
8026
|
+
view = self._reqView()
|
|
8027
|
+
|
|
8028
|
+
self.runt.reqAdmin(gateiden=view.iden)
|
|
8029
|
+
self.runt.confirm(('layer', 'del'), gateiden=view.layers[0].iden)
|
|
8030
|
+
|
|
8031
|
+
await view.swapLayer()
|
|
8032
|
+
|
|
8020
8033
|
async def getMerges(self):
|
|
8021
8034
|
view = self._reqView()
|
|
8022
8035
|
async for merge in view.getMerges():
|
synapse/lib/trigger.py
CHANGED
|
@@ -494,7 +494,7 @@ class Trigger:
|
|
|
494
494
|
await self.view.core.getStormQuery(valu)
|
|
495
495
|
|
|
496
496
|
self.tdef[name] = valu
|
|
497
|
-
|
|
497
|
+
self.view.trigdict.set(self.iden, self.tdef)
|
|
498
498
|
|
|
499
499
|
def get(self, name):
|
|
500
500
|
return self.tdef.get(name)
|
synapse/lib/version.py
CHANGED
|
@@ -223,6 +223,6 @@ def reqVersion(valu, reqver,
|
|
|
223
223
|
##############################################################################
|
|
224
224
|
# The following are touched during the release process by bumpversion.
|
|
225
225
|
# Do not modify these directly.
|
|
226
|
-
version = (2,
|
|
226
|
+
version = (2, 177, 0)
|
|
227
227
|
verstring = '.'.join([str(x) for x in version])
|
|
228
|
-
commit = '
|
|
228
|
+
commit = 'b7da795aeb690020e38e99dc5aa1f505cdc2d659'
|
synapse/lib/view.py
CHANGED
|
@@ -79,18 +79,17 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
79
79
|
'''
|
|
80
80
|
snapctor = s_snap.Snap.anit
|
|
81
81
|
|
|
82
|
-
async def __anit__(self, core,
|
|
82
|
+
async def __anit__(self, core, vdef):
|
|
83
83
|
'''
|
|
84
84
|
Async init the view.
|
|
85
85
|
|
|
86
86
|
Args:
|
|
87
87
|
core (Cortex): The cortex that owns the view.
|
|
88
|
-
|
|
88
|
+
vdef (dict): The dictionary containing the view definition.
|
|
89
89
|
'''
|
|
90
|
-
self.
|
|
91
|
-
self.iden = node.name()
|
|
90
|
+
self.iden = vdef.get('iden')
|
|
92
91
|
self.bidn = s_common.uhex(self.iden)
|
|
93
|
-
self.info =
|
|
92
|
+
self.info = vdef
|
|
94
93
|
|
|
95
94
|
self.core = core
|
|
96
95
|
self.dirn = s_common.gendir(core.dirn, 'views', self.iden)
|
|
@@ -101,11 +100,10 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
101
100
|
|
|
102
101
|
self.trigqueue = self.viewslab.getSeqn('trigqueue')
|
|
103
102
|
|
|
104
|
-
|
|
105
|
-
self.trigdict = await trignode.dict()
|
|
103
|
+
self.trigdict = self.core.cortexdata.getSubKeyVal(f'view:{self.iden}:trigger:')
|
|
106
104
|
|
|
107
105
|
self.triggers = s_trigger.Triggers(self)
|
|
108
|
-
for
|
|
106
|
+
for tdef in self.trigdict.values():
|
|
109
107
|
try:
|
|
110
108
|
await self.triggers.load(tdef)
|
|
111
109
|
|
|
@@ -268,12 +266,14 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
268
266
|
return
|
|
269
267
|
|
|
270
268
|
self.merging = True
|
|
271
|
-
|
|
269
|
+
self.info['merging'] = True
|
|
270
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
272
271
|
|
|
273
272
|
layr = self.layers[0]
|
|
274
273
|
|
|
275
274
|
layr.readonly = True
|
|
276
|
-
|
|
275
|
+
layr.layrinfo['readonly'] = True
|
|
276
|
+
self.core.layerdefs.set(layr.iden, layr.layrinfo)
|
|
277
277
|
|
|
278
278
|
merge = self.getMergeRequest()
|
|
279
279
|
votes = [vote async for vote in self.getMergeVotes()]
|
|
@@ -490,10 +490,11 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
490
490
|
await self._delMergeMeta()
|
|
491
491
|
|
|
492
492
|
self.parent = None
|
|
493
|
-
|
|
493
|
+
if self.info.pop('parent', None) is not None:
|
|
494
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
494
495
|
|
|
495
|
-
|
|
496
|
-
|
|
496
|
+
await self.core.feedBeholder('view:set', {'iden': self.iden, 'name': 'parent', 'valu': None},
|
|
497
|
+
gates=[self.iden, self.layers[0].iden])
|
|
497
498
|
|
|
498
499
|
async def mergeStormIface(self, name, todo):
|
|
499
500
|
'''
|
|
@@ -707,15 +708,17 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
707
708
|
# Add all of the bottom view's layers.
|
|
708
709
|
layers.extend(view.layers)
|
|
709
710
|
|
|
710
|
-
self.layers = layers
|
|
711
711
|
layridens = [layr.iden for layr in layers]
|
|
712
|
-
await self.info.set('layers', layridens)
|
|
713
712
|
|
|
714
|
-
|
|
713
|
+
self.layers = layers
|
|
714
|
+
self.info['layers'] = layridens
|
|
715
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
716
|
+
|
|
717
|
+
await self.core.feedBeholder('view:setlayers', {'iden': self.iden, 'layers': layridens}, gates=[self.iden, self.layers[0].iden])
|
|
715
718
|
|
|
716
719
|
async def pack(self):
|
|
717
720
|
d = {'iden': self.iden}
|
|
718
|
-
d.update(self.info
|
|
721
|
+
d.update(self.info)
|
|
719
722
|
|
|
720
723
|
layrinfo = [await lyr.pack() for lyr in self.layers]
|
|
721
724
|
d['layers'] = layrinfo
|
|
@@ -855,6 +858,7 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
855
858
|
|
|
856
859
|
async def _initViewLayers(self):
|
|
857
860
|
|
|
861
|
+
self.layers = []
|
|
858
862
|
for iden in self.info.get('layers'):
|
|
859
863
|
|
|
860
864
|
layr = self.core.layers.get(iden)
|
|
@@ -1121,7 +1125,8 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1121
1125
|
raise s_exc.BadArg(mesg=mesg)
|
|
1122
1126
|
|
|
1123
1127
|
self.parent = parent
|
|
1124
|
-
|
|
1128
|
+
self.info[name] = valu
|
|
1129
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
1125
1130
|
|
|
1126
1131
|
await self._calcForkLayers()
|
|
1127
1132
|
|
|
@@ -1137,7 +1142,7 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1137
1142
|
# enforce ( which will need to be done very carefully to prevent
|
|
1138
1143
|
# existing non-compliant values from causing issues with existing views )
|
|
1139
1144
|
if valu is not None:
|
|
1140
|
-
vdef = self.info
|
|
1145
|
+
vdef = s_msgpack.deepcopy(self.info)
|
|
1141
1146
|
vdef['quorum'] = s_msgpack.deepcopy(valu)
|
|
1142
1147
|
s_schemas.reqValidView(vdef)
|
|
1143
1148
|
else:
|
|
@@ -1148,14 +1153,24 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1148
1153
|
await view._delMergeRequest()
|
|
1149
1154
|
|
|
1150
1155
|
if valu is None:
|
|
1151
|
-
|
|
1156
|
+
self.info.pop(name, None)
|
|
1152
1157
|
else:
|
|
1153
|
-
|
|
1158
|
+
self.info[name] = valu
|
|
1159
|
+
|
|
1160
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
1154
1161
|
|
|
1155
1162
|
await self.core.feedBeholder('view:set', {'iden': self.iden, 'name': name, 'valu': valu},
|
|
1156
1163
|
gates=[self.iden, self.layers[0].iden])
|
|
1157
1164
|
return valu
|
|
1158
1165
|
|
|
1166
|
+
async def _setLayerIdens(self, idens):
|
|
1167
|
+
# this may only be called from within a nexus handler...
|
|
1168
|
+
self.info['layers'] = idens
|
|
1169
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
1170
|
+
await self._initViewLayers()
|
|
1171
|
+
await self.core.feedBeholder('view:set', {'iden': self.iden, 'name': 'layers', 'valu': idens},
|
|
1172
|
+
gates=[self.iden, self.layers[0].iden])
|
|
1173
|
+
|
|
1159
1174
|
async def addLayer(self, layriden, indx=None):
|
|
1160
1175
|
if any(layriden == layr.iden for layr in self.layers):
|
|
1161
1176
|
raise s_exc.DupIden(mesg='May not have the same layer in a view twice')
|
|
@@ -1184,7 +1199,9 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1184
1199
|
else:
|
|
1185
1200
|
self.layers.insert(indx, layr)
|
|
1186
1201
|
|
|
1187
|
-
|
|
1202
|
+
self.info['layers'] = [lyr.iden for lyr in self.layers]
|
|
1203
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
1204
|
+
|
|
1188
1205
|
await self.core.feedBeholder('view:addlayer', {'iden': self.iden, 'layer': layriden, 'indx': indx}, gates=[self.iden, layriden])
|
|
1189
1206
|
self.core._calcViewsByLayer()
|
|
1190
1207
|
|
|
@@ -1212,7 +1229,9 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1212
1229
|
self.invalid = None
|
|
1213
1230
|
self.layers = layrs
|
|
1214
1231
|
|
|
1215
|
-
|
|
1232
|
+
self.info['layers'] = layers
|
|
1233
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
1234
|
+
|
|
1216
1235
|
await self.core.feedBeholder('view:setlayers', {'iden': self.iden, 'layers': layers}, gates=[self.iden, layers[0]])
|
|
1217
1236
|
|
|
1218
1237
|
await self._calcChildViews()
|
|
@@ -1242,7 +1261,8 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1242
1261
|
|
|
1243
1262
|
# convert layers to a list of idens...
|
|
1244
1263
|
lids = [layr.iden for layr in layers]
|
|
1245
|
-
|
|
1264
|
+
child.info['layers'] = lids
|
|
1265
|
+
self.core.viewdefs.set(child.iden, child.info)
|
|
1246
1266
|
|
|
1247
1267
|
await self.core.feedBeholder('view:setlayers', {'iden': child.iden, 'layers': lids}, gates=[child.iden, lids[0]])
|
|
1248
1268
|
|
|
@@ -1300,8 +1320,9 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1300
1320
|
await self.core._addView(vdef)
|
|
1301
1321
|
|
|
1302
1322
|
forkiden = vdef.get('iden')
|
|
1323
|
+
self.info['parent'] = forkiden
|
|
1303
1324
|
self.parent = self.core.reqView(forkiden)
|
|
1304
|
-
|
|
1325
|
+
self.core.viewdefs.set(self.iden, self.info)
|
|
1305
1326
|
|
|
1306
1327
|
mesg = {'iden': self.iden, 'name': 'parent', 'valu': forkiden}
|
|
1307
1328
|
await self.core.feedBeholder('view:set', mesg, gates=[self.iden, self.layers[0].iden])
|
|
@@ -1386,6 +1407,12 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1386
1407
|
async for nodeedits in fromlayr.iterLayerNodeEdits():
|
|
1387
1408
|
await self.parent.storNodeEdits([nodeedits], meta)
|
|
1388
1409
|
|
|
1410
|
+
async def swapLayer(self):
|
|
1411
|
+
oldlayr = self.layers[0]
|
|
1412
|
+
newlayr = await self.core._twinLayer(oldlayr)
|
|
1413
|
+
await self.core.swapLayer(oldlayr.iden, newlayr.iden)
|
|
1414
|
+
await self.core.delLayer(oldlayr.iden)
|
|
1415
|
+
|
|
1389
1416
|
async def wipeLayer(self, useriden=None):
|
|
1390
1417
|
'''
|
|
1391
1418
|
Delete the data in the write layer by generating del nodeedits.
|
|
@@ -1549,7 +1576,7 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1549
1576
|
|
|
1550
1577
|
trig = await self.triggers.load(tdef)
|
|
1551
1578
|
|
|
1552
|
-
|
|
1579
|
+
self.trigdict.set(trig.iden, tdef)
|
|
1553
1580
|
await self.core.auth.addAuthGate(trig.iden, 'trigger')
|
|
1554
1581
|
await user.setAdmin(True, gateiden=tdef.get('iden'), logged=False)
|
|
1555
1582
|
|
|
@@ -1581,7 +1608,7 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1581
1608
|
return
|
|
1582
1609
|
|
|
1583
1610
|
await self.core.feedBeholder('trigger:del', {'iden': trig.iden, 'view': trig.view.iden}, gates=[trig.iden])
|
|
1584
|
-
|
|
1611
|
+
self.trigdict.pop(trig.iden)
|
|
1585
1612
|
await self.core.auth.delAuthGate(trig.iden)
|
|
1586
1613
|
|
|
1587
1614
|
@s_nexus.Pusher.onPushAuto('trigger:set')
|
|
@@ -1607,7 +1634,7 @@ class View(s_nexus.Pusher): # type: ignore
|
|
|
1607
1634
|
Note: this does not delete any layer storage.
|
|
1608
1635
|
'''
|
|
1609
1636
|
await self.fini()
|
|
1610
|
-
await self.
|
|
1637
|
+
await self.trigdict.truncate()
|
|
1611
1638
|
await self._wipeViewMeta()
|
|
1612
1639
|
shutil.rmtree(self.dirn, ignore_errors=True)
|
|
1613
1640
|
|
synapse/models/base.py
CHANGED
|
@@ -59,6 +59,10 @@ class BaseModule(s_module.CoreModule):
|
|
|
59
59
|
('meta:ruleset', ('guid', {}), {
|
|
60
60
|
'doc': 'A set of rules linked with -(has)> edges.'}),
|
|
61
61
|
|
|
62
|
+
('meta:rule:type:taxonomy', ('taxonomy', {}), {
|
|
63
|
+
'interfaces': ('meta:taxonomy',),
|
|
64
|
+
'doc': 'A taxonomy for meta:rule types.'}),
|
|
65
|
+
|
|
62
66
|
('meta:rule', ('guid', {}), {
|
|
63
67
|
'doc': 'A generic rule linked to matches with -(matches)> edges.'}),
|
|
64
68
|
|
|
@@ -257,9 +261,12 @@ class BaseModule(s_module.CoreModule):
|
|
|
257
261
|
'doc': 'The time the ruleset was most recently modified.'}),
|
|
258
262
|
)),
|
|
259
263
|
|
|
264
|
+
('meta:rule:type:taxonomy', {}, ()),
|
|
260
265
|
('meta:rule', {}, (
|
|
261
266
|
('name', ('str', {'lower': True, 'onespace': True}), {
|
|
262
267
|
'doc': 'A name for the rule.'}),
|
|
268
|
+
('type', ('meta:rule:type:taxonomy', {}), {
|
|
269
|
+
'doc': 'The rule type.'}),
|
|
263
270
|
('desc', ('str', {}), {
|
|
264
271
|
'disp': {'hint': 'text'},
|
|
265
272
|
'doc': 'A description of the rule.'}),
|