neuralui 0.12.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- neuralui/__init__.py +0 -0
- neuralui/__main__.py +7 -0
- neuralui/actions.py +504 -0
- neuralui/checkpkg.py +60 -0
- neuralui/export.py +103 -0
- neuralui/handlers.py +1885 -0
- neuralui/jobs.py +400 -0
- neuralui/limits.py +71 -0
- neuralui/lint.py +508 -0
- neuralui/pkg.py +27 -0
- neuralui/pkgproto/neuralui.yaml +195 -0
- neuralui/pkgproto/storm/commands/neuralui.query.migrate.storm +12 -0
- neuralui/pkgproto/storm/commands/neuralui.query.save.storm +1 -0
- neuralui/pkgproto/storm/commands/neuralui.scrape.migrate.storm +20 -0
- neuralui/pkgproto/storm/modules/neuralui.storm +121 -0
- neuralui/schema/neuralui.schema.json +144 -0
- neuralui/security.py +265 -0
- neuralui/service.py +342 -0
- neuralui/store.py +399 -0
- neuralui/version.py +12 -0
- neuralui/web/brand/neuralui-compass-primary-180.png +0 -0
- neuralui/web/brand/neuralui-compass-primary-light.svg +1 -0
- neuralui/web/brand/neuralui-compass-primary.svg +1 -0
- neuralui/web/brand/neuralui-compass-small-32.png +0 -0
- neuralui/web/brand/neuralui-compass-small-light.svg +1 -0
- neuralui/web/brand/neuralui-compass-small.svg +1 -0
- neuralui/web/css/app.css +1420 -0
- neuralui/web/docs/actions.md +137 -0
- neuralui/web/docs/admin.md +45 -0
- neuralui/web/docs/automation.md +48 -0
- neuralui/web/docs/console.md +25 -0
- neuralui/web/docs/details.md +55 -0
- neuralui/web/docs/explore.md +11 -0
- neuralui/web/docs/graph.md +71 -0
- neuralui/web/docs/history.md +83 -0
- neuralui/web/docs/img/about.png +0 -0
- neuralui/web/docs/img/action-palette.png +0 -0
- neuralui/web/docs/img/add-page.png +0 -0
- neuralui/web/docs/img/admin-actions.png +0 -0
- neuralui/web/docs/img/applicator.png +0 -0
- neuralui/web/docs/img/automation.png +0 -0
- neuralui/web/docs/img/bulkedit.png +0 -0
- neuralui/web/docs/img/command-dialog.png +0 -0
- neuralui/web/docs/img/completion.png +0 -0
- neuralui/web/docs/img/drill.png +0 -0
- neuralui/web/docs/img/enrich-menu.png +0 -0
- neuralui/web/docs/img/error-state.png +0 -0
- neuralui/web/docs/img/graph-large.png +0 -0
- neuralui/web/docs/img/graph-okabe-ito.png +0 -0
- neuralui/web/docs/img/graph.png +0 -0
- neuralui/web/docs/img/guide.png +0 -0
- neuralui/web/docs/img/history.png +0 -0
- neuralui/web/docs/img/import.png +0 -0
- neuralui/web/docs/img/jobs-panel.png +0 -0
- neuralui/web/docs/img/lint.png +0 -0
- neuralui/web/docs/img/menu.png +0 -0
- neuralui/web/docs/img/note.png +0 -0
- neuralui/web/docs/img/open-in.png +0 -0
- neuralui/web/docs/img/output-view.png +0 -0
- neuralui/web/docs/img/palette.png +0 -0
- neuralui/web/docs/img/scrape-results.png +0 -0
- neuralui/web/docs/img/scrape-review.png +0 -0
- neuralui/web/docs/img/script.png +0 -0
- neuralui/web/docs/img/table.png +0 -0
- neuralui/web/docs/img/tagnode.png +0 -0
- neuralui/web/docs/img/tags.png +0 -0
- neuralui/web/docs/img/theme-high-contrast.png +0 -0
- neuralui/web/docs/img/theme-light.png +0 -0
- neuralui/web/docs/img/update-banner.png +0 -0
- neuralui/web/docs/img/viewer-hex.png +0 -0
- neuralui/web/docs/import.md +57 -0
- neuralui/web/docs/keyboard.md +45 -0
- neuralui/web/docs/library.md +49 -0
- neuralui/web/docs/overview.md +76 -0
- neuralui/web/docs/queries.md +103 -0
- neuralui/web/docs/results.md +103 -0
- neuralui/web/docs/scrape.md +39 -0
- neuralui/web/docs/settings.md +94 -0
- neuralui/web/docs/tags.md +88 -0
- neuralui/web/index.html +21 -0
- neuralui/web/js/actionpalette.js +99 -0
- neuralui/web/js/actions.js +165 -0
- neuralui/web/js/addpage.js +62 -0
- neuralui/web/js/admin.js +548 -0
- neuralui/web/js/api.js +316 -0
- neuralui/web/js/app.js +529 -0
- neuralui/web/js/automation.js +723 -0
- neuralui/web/js/boot.js +7 -0
- neuralui/web/js/complete.js +272 -0
- neuralui/web/js/console.js +204 -0
- neuralui/web/js/dialog.js +267 -0
- neuralui/web/js/drill.js +301 -0
- neuralui/web/js/editor.js +725 -0
- neuralui/web/js/editors.js +640 -0
- neuralui/web/js/explore.js +161 -0
- neuralui/web/js/export.js +222 -0
- neuralui/web/js/files.js +85 -0
- neuralui/web/js/graph.js +530 -0
- neuralui/web/js/help.js +35 -0
- neuralui/web/js/helppage.js +304 -0
- neuralui/web/js/import.js +408 -0
- neuralui/web/js/importfiles.js +262 -0
- neuralui/web/js/jobs.js +211 -0
- neuralui/web/js/layout-worker.js +107 -0
- neuralui/web/js/livemd.js +509 -0
- neuralui/web/js/lookups.js +81 -0
- neuralui/web/js/menu.js +752 -0
- neuralui/web/js/model.js +267 -0
- neuralui/web/js/notepage.js +111 -0
- neuralui/web/js/notes.js +363 -0
- neuralui/web/js/page.js +91 -0
- neuralui/web/js/palette.js +128 -0
- neuralui/web/js/queries.js +163 -0
- neuralui/web/js/results.js +1006 -0
- neuralui/web/js/saved.js +250 -0
- neuralui/web/js/scrape.js +328 -0
- neuralui/web/js/script.js +373 -0
- neuralui/web/js/state.js +268 -0
- neuralui/web/js/tabs.js +1118 -0
- neuralui/web/js/tags.js +114 -0
- neuralui/web/js/undo.js +104 -0
- neuralui/web/js/util.js +195 -0
- neuralui/web/js/viewer.js +437 -0
- neuralui/web/vendor/cytoscape.LICENSE +19 -0
- neuralui/web/vendor/cytoscape.min.js +32 -0
- neuralui/web/vendor/dompurify.LICENSE +202 -0
- neuralui/web/vendor/inter/OFL.txt +93 -0
- neuralui/web/vendor/inter/inter-latin-ext-wght-normal.woff2 +0 -0
- neuralui/web/vendor/inter/inter-latin-wght-normal.woff2 +0 -0
- neuralui/web/vendor/marked.LICENSE +44 -0
- neuralui/web/vendor/marked.min.js +6 -0
- neuralui/web/vendor/purify.min.js +7 -0
- neuralui-0.12.2.dist-info/METADATA +679 -0
- neuralui-0.12.2.dist-info/RECORD +136 -0
- neuralui-0.12.2.dist-info/WHEEL +5 -0
- neuralui-0.12.2.dist-info/top_level.txt +1 -0
neuralui/__init__.py
ADDED
|
File without changes
|
neuralui/__main__.py
ADDED
neuralui/actions.py
ADDED
|
@@ -0,0 +1,504 @@
|
|
|
1
|
+
'''
|
|
2
|
+
The right-click action registry: every node action the UI offers, built from the Storm packages in the Cortex.
|
|
3
|
+
|
|
4
|
+
Actions come from four places (docs/PACKAGES.md describes the formats, neuralui/schema/neuralui.schema.json
|
|
5
|
+
is the JSON Schema of the neuralui: key):
|
|
6
|
+
|
|
7
|
+
neuralui a package's neuralui: actions: (query, newtab, filter, storm, link and command actions)
|
|
8
|
+
optic a Rapid Power-Up's optic: actions:, shown as Optic shows them (Actions > <package>)
|
|
9
|
+
command every package command which says what forms it takes (Enrich > <package>)
|
|
10
|
+
defaults the neuralui package's own actions (pivots, filters, Open in ...), even before service.add
|
|
11
|
+
|
|
12
|
+
buildRegistry() turns package definitions into the list of actions (with the entries it could not use, and
|
|
13
|
+
why, for the Admin page). An admin's settings (actions hidden for everyone or for roles, default pins) are
|
|
14
|
+
applied per user by visibleActions(); the user's permissions are checked by the HTTP handler.
|
|
15
|
+
'''
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
import json
|
|
19
|
+
import hashlib
|
|
20
|
+
import functools
|
|
21
|
+
|
|
22
|
+
import synapse.exc as s_exc
|
|
23
|
+
import synapse.lib.config as s_config
|
|
24
|
+
|
|
25
|
+
# a link action's templates must name a web page: no javascript:, data: or other schemes
|
|
26
|
+
LINK_PREFIXES = ('https://', 'http://')
|
|
27
|
+
|
|
28
|
+
ACTION_TYPES = ('query', 'newtab', 'filter', 'storm', 'link', 'command')
|
|
29
|
+
|
|
30
|
+
# menu: where an action goes at the top of the menu (its group, if it has one, becomes the submenu inside)
|
|
31
|
+
MENUS = {
|
|
32
|
+
'pivot': 'Pivot',
|
|
33
|
+
'filter': 'Filter',
|
|
34
|
+
'enrich': 'Enrich',
|
|
35
|
+
'edit': 'Edit',
|
|
36
|
+
'open': 'Open in',
|
|
37
|
+
'actions': 'Actions',
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
YIELDS = ('inbound', 'new', 'none')
|
|
41
|
+
|
|
42
|
+
# the version of the neuralui: package format this NeuralUI reads (a package's neuralui: version:, default 1).
|
|
43
|
+
# A newer minor NeuralUI may add optional keys; a package which needs a newer format says so with version:.
|
|
44
|
+
FORMAT_VERSION = 1
|
|
45
|
+
|
|
46
|
+
# the keys NeuralUI reads. Others are ignored (so a package written for a newer 1.x still loads) and listed
|
|
47
|
+
# as warnings in Admin > Actions, where a typo shows up.
|
|
48
|
+
PKG_KEYS = ('version', 'actions', 'notetypes', 'hide', 'commands')
|
|
49
|
+
ACTION_KEYS = ('id', 'name', 'type', 'forms', 'group', 'menu', 'storm', 'url', 'command', 'values', 'args',
|
|
50
|
+
'descr', 'multi', 'confirm', 'perms', 'max', 'batch', 'yields', 'background')
|
|
51
|
+
NOTETYPE_KEYS = ('type', 'title', 'render', 'descr')
|
|
52
|
+
NOTE_RENDERS = ('markdown', 'text')
|
|
53
|
+
|
|
54
|
+
# an action id: stable within its package, and part of its iden (<pkg>:<id>), so no colons
|
|
55
|
+
ID_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$')
|
|
56
|
+
|
|
57
|
+
# the most nodes, and batch size, a storm or command action may set (the page defaults to 500 and 100)
|
|
58
|
+
MAX_MAX = 100000
|
|
59
|
+
|
|
60
|
+
SCHEMA_PATH = os.path.join(os.path.dirname(__file__), 'schema', 'neuralui.schema.json')
|
|
61
|
+
|
|
62
|
+
@functools.cache
|
|
63
|
+
def _validator():
|
|
64
|
+
with open(SCHEMA_PATH) as fd:
|
|
65
|
+
return s_config.getJsValidator(json.load(fd))
|
|
66
|
+
|
|
67
|
+
def checkSchema(info):
|
|
68
|
+
'''
|
|
69
|
+
Check a package's neuralui: key against the JSON Schema (stricter than NeuralUI, which ignores unknown
|
|
70
|
+
keys). Returns the problem as a message, or None when it is valid.
|
|
71
|
+
'''
|
|
72
|
+
try:
|
|
73
|
+
_validator()(json.loads(json.dumps(info)))
|
|
74
|
+
except s_exc.SchemaViolation as e:
|
|
75
|
+
return e.get('mesg') or str(e)
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
class ActionError(Exception):
|
|
79
|
+
'''An action entry which can not be used: the message says why (shown in Admin > Actions).'''
|
|
80
|
+
|
|
81
|
+
def _strs(valu):
|
|
82
|
+
'''A list of strings from a list or tuple (packages read back from the Cortex have tuples), or None.'''
|
|
83
|
+
if not isinstance(valu, (list, tuple)):
|
|
84
|
+
return None
|
|
85
|
+
return [v for v in valu if isinstance(v, str)]
|
|
86
|
+
|
|
87
|
+
def _posint(valu, most=MAX_MAX):
|
|
88
|
+
return valu if isinstance(valu, int) and not isinstance(valu, bool) and 0 < valu <= most else None
|
|
89
|
+
|
|
90
|
+
def neuralAction(adef):
|
|
91
|
+
'''
|
|
92
|
+
A package's neuralui action, checked, with its defaults. Raises ActionError when it can not be used.
|
|
93
|
+
|
|
94
|
+
A link action (type: link) opens a web page for a node instead of running Storm: its url is a template,
|
|
95
|
+
or a {form: template} map, or a {form: [template, ...]} map tried in order (see docs/PACKAGES.md).
|
|
96
|
+
'''
|
|
97
|
+
if not isinstance(adef, dict):
|
|
98
|
+
raise ActionError('An action must be an object.')
|
|
99
|
+
if not isinstance(adef.get('name'), str) or not adef['name'].strip():
|
|
100
|
+
raise ActionError('An action needs a name.')
|
|
101
|
+
adef = dict(adef)
|
|
102
|
+
|
|
103
|
+
forms = _strs(adef.get('forms', ['*']))
|
|
104
|
+
if not forms:
|
|
105
|
+
raise ActionError('forms must be a list of form names or globs.')
|
|
106
|
+
adef['forms'] = forms
|
|
107
|
+
|
|
108
|
+
if adef.get('id') is not None and (not isinstance(adef['id'], str) or not ID_RE.match(adef['id'])):
|
|
109
|
+
raise ActionError('id must be letters, digits, dots, dashes and underscores (up to 128), starting with a letter or digit.')
|
|
110
|
+
|
|
111
|
+
kind = adef.setdefault('type', 'command' if isinstance(adef.get('command'), str) else 'query')
|
|
112
|
+
if kind not in ACTION_TYPES:
|
|
113
|
+
raise ActionError(f'Unknown type {kind!r} (one of {", ".join(ACTION_TYPES)}).')
|
|
114
|
+
|
|
115
|
+
# where it goes in the menu: menu picks the top level, and a group becomes the submenu inside it
|
|
116
|
+
menu = adef.pop('menu', None)
|
|
117
|
+
if menu is not None:
|
|
118
|
+
if menu not in MENUS:
|
|
119
|
+
raise ActionError(f'Unknown menu {menu!r} (one of {", ".join(MENUS)}).')
|
|
120
|
+
if adef.get('group') and not adef.get('subgroup'):
|
|
121
|
+
adef['subgroup'] = adef['group']
|
|
122
|
+
adef['group'] = MENUS[menu]
|
|
123
|
+
|
|
124
|
+
for name in ('max', 'batch'):
|
|
125
|
+
if adef.get(name) is not None:
|
|
126
|
+
if _posint(adef[name]) is None:
|
|
127
|
+
raise ActionError(f'{name} must be a whole number from 1 to {MAX_MAX}.')
|
|
128
|
+
if adef.get('yields') is not None and adef['yields'] not in YIELDS:
|
|
129
|
+
raise ActionError(f'yields must be one of {", ".join(YIELDS)}.')
|
|
130
|
+
if adef.get('perms') is not None:
|
|
131
|
+
perms = permList(adef['perms'])
|
|
132
|
+
if perms is None:
|
|
133
|
+
raise ActionError('perms must be a list of permission paths, such as [["acme", "user"]].')
|
|
134
|
+
adef['perms'] = perms
|
|
135
|
+
|
|
136
|
+
if kind == 'link':
|
|
137
|
+
url = adef.get('url')
|
|
138
|
+
if isinstance(url, str):
|
|
139
|
+
url = {'*': [url]}
|
|
140
|
+
if not isinstance(url, dict) or not url:
|
|
141
|
+
raise ActionError('A link action needs a url: a template, or a map of form to templates.')
|
|
142
|
+
urls = {}
|
|
143
|
+
for form, temps in url.items():
|
|
144
|
+
temps = [temps] if isinstance(temps, str) else temps
|
|
145
|
+
if not isinstance(form, str) or not isinstance(temps, (list, tuple)) or not temps:
|
|
146
|
+
raise ActionError('A link url maps a form to a template or a list of templates.')
|
|
147
|
+
for temp in temps:
|
|
148
|
+
if not isinstance(temp, str) or not temp.startswith(LINK_PREFIXES):
|
|
149
|
+
raise ActionError(f'A link template must start with https:// or http:// ({temp!r} does not).')
|
|
150
|
+
urls[form] = list(temps)
|
|
151
|
+
adef['url'] = urls
|
|
152
|
+
if adef['forms'] == ['*']:
|
|
153
|
+
adef['forms'] = list(urls)
|
|
154
|
+
# a link opens one browser tab: it is for one node at a time
|
|
155
|
+
adef['multi'] = False
|
|
156
|
+
return adef
|
|
157
|
+
|
|
158
|
+
if kind == 'command':
|
|
159
|
+
# runs one of the package's commands (its args, forms and perms come from the command)
|
|
160
|
+
if not isinstance(adef.get('command'), str) or not adef['command']:
|
|
161
|
+
raise ActionError('A command action needs the name of one of the package\'s commands.')
|
|
162
|
+
if adef.get('values') is not None and not isinstance(adef['values'], dict):
|
|
163
|
+
raise ActionError('values must map argument names to values.')
|
|
164
|
+
return adef
|
|
165
|
+
|
|
166
|
+
if not isinstance(adef.get('storm'), str) or not adef['storm'].strip():
|
|
167
|
+
raise ActionError(f'A {kind} action needs storm.')
|
|
168
|
+
return adef
|
|
169
|
+
|
|
170
|
+
def cmdForms(cdef):
|
|
171
|
+
'''The forms a Storm command takes as input: cmdinputs, or the older forms: input: hints.'''
|
|
172
|
+
forms = []
|
|
173
|
+
for item in cdef.get('cmdinputs') or ():
|
|
174
|
+
if isinstance(item, dict) and isinstance(item.get('form'), str):
|
|
175
|
+
forms.append(item['form'])
|
|
176
|
+
hints = cdef.get('forms')
|
|
177
|
+
if not forms and isinstance(hints, dict):
|
|
178
|
+
forms = [f for f in hints.get('input') or () if isinstance(f, str)]
|
|
179
|
+
return forms
|
|
180
|
+
|
|
181
|
+
def cmdYields(cdef):
|
|
182
|
+
'''
|
|
183
|
+
What a command's output is for the results: 'new' when it says it yields forms it does not take (show
|
|
184
|
+
them in a new tab), else 'inbound' (the inbound nodes, updated in place).
|
|
185
|
+
'''
|
|
186
|
+
hints = cdef.get('forms')
|
|
187
|
+
output = set(_strs(hints.get('output')) or ()) if isinstance(hints, dict) else set()
|
|
188
|
+
return 'new' if output - set(cmdForms(cdef)) else 'inbound'
|
|
189
|
+
|
|
190
|
+
def cmdFields(cmdargs):
|
|
191
|
+
'''
|
|
192
|
+
A Storm command's arguments (cmdargs, argparse style) as prompt fields. Each field keeps how it goes on
|
|
193
|
+
the command line: flag (None for a positional argument), and for a list whether the flag repeats per
|
|
194
|
+
value (append) or takes them all (nargs). A store_true / store_false flag becomes a checkbox.
|
|
195
|
+
'''
|
|
196
|
+
fields = []
|
|
197
|
+
for item in cmdargs or ():
|
|
198
|
+
if not isinstance(item, (list, tuple)) or not item or not isinstance(item[0], str):
|
|
199
|
+
continue
|
|
200
|
+
name = item[0]
|
|
201
|
+
opts = item[1] if len(item) > 1 and isinstance(item[1], dict) else {}
|
|
202
|
+
if name in ('-h', '--help'):
|
|
203
|
+
continue
|
|
204
|
+
positional = not name.startswith('-')
|
|
205
|
+
action = opts.get('action')
|
|
206
|
+
nargs = opts.get('nargs')
|
|
207
|
+
helptext = opts.get('help') if isinstance(opts.get('help'), str) else ''
|
|
208
|
+
field = {
|
|
209
|
+
'name': opts.get('dest') if isinstance(opts.get('dest'), str) else name.lstrip('-').replace('-', '_'),
|
|
210
|
+
'label': name,
|
|
211
|
+
'flag': None if positional else name,
|
|
212
|
+
'help': helptext.strip(),
|
|
213
|
+
}
|
|
214
|
+
default = opts.get('default')
|
|
215
|
+
if action in ('store_true', 'store_false'):
|
|
216
|
+
field['type'] = 'bool'
|
|
217
|
+
field['action'] = action
|
|
218
|
+
# the box shows what the command does when the flag is left out
|
|
219
|
+
field['default'] = action == 'store_false'
|
|
220
|
+
elif action == 'append' or nargs in ('*', '+') or (isinstance(nargs, int) and nargs > 1):
|
|
221
|
+
field['type'] = 'list'
|
|
222
|
+
field['mode'] = 'append' if action == 'append' else 'nargs'
|
|
223
|
+
elif isinstance(opts.get('choices'), (list, tuple)) and opts['choices']:
|
|
224
|
+
field['type'] = 'select'
|
|
225
|
+
choices = [c for c in opts['choices'] if isinstance(c, (str, int, float))]
|
|
226
|
+
field['options'] = ([] if positional else [['', '(default)']]) + [[str(c), str(c)] for c in choices]
|
|
227
|
+
elif opts.get('type') == 'int':
|
|
228
|
+
field['type'] = 'int'
|
|
229
|
+
else:
|
|
230
|
+
field['type'] = 'str'
|
|
231
|
+
if field['type'] != 'bool' and isinstance(default, (str, int, float)) and not isinstance(default, bool):
|
|
232
|
+
field['default'] = default
|
|
233
|
+
if field['type'] == 'list' and isinstance(default, (list, tuple)):
|
|
234
|
+
field['default'] = [str(d) for d in default]
|
|
235
|
+
if (positional and nargs not in ('?', '*')) or opts.get('required') is True:
|
|
236
|
+
field['required'] = True
|
|
237
|
+
fields.append(field)
|
|
238
|
+
return fields
|
|
239
|
+
|
|
240
|
+
def presetFields(fields, values):
|
|
241
|
+
'''A command action's fields with its preset values as their defaults (the dialog still shows them).'''
|
|
242
|
+
if not values:
|
|
243
|
+
return fields
|
|
244
|
+
out = []
|
|
245
|
+
for field in fields:
|
|
246
|
+
if field['name'] in values:
|
|
247
|
+
field = dict(field)
|
|
248
|
+
valu = values[field['name']]
|
|
249
|
+
if field['type'] == 'list' and isinstance(valu, (list, tuple)):
|
|
250
|
+
valu = [str(v) for v in valu]
|
|
251
|
+
field['default'] = valu
|
|
252
|
+
out.append(field)
|
|
253
|
+
return out
|
|
254
|
+
|
|
255
|
+
def firstPara(text):
|
|
256
|
+
'''The first paragraph of a description, for a tooltip.'''
|
|
257
|
+
if not isinstance(text, str):
|
|
258
|
+
return None
|
|
259
|
+
return text.strip().split('\n\n', 1)[0].strip() or None
|
|
260
|
+
|
|
261
|
+
def permList(perms):
|
|
262
|
+
'''A command's perms as a list of perm paths (lists of names), or None.'''
|
|
263
|
+
if not isinstance(perms, (list, tuple)):
|
|
264
|
+
return None
|
|
265
|
+
perms = [list(p) for p in perms if isinstance(p, (list, tuple)) and p and all(isinstance(n, str) for n in p)]
|
|
266
|
+
return perms or None
|
|
267
|
+
|
|
268
|
+
def opticAction(pkgname, adef):
|
|
269
|
+
'''An Optic action (optic: actions:) as Optic shows it: Actions > <package> > <name>, run as storm.'''
|
|
270
|
+
if not isinstance(adef, dict) or not isinstance(adef.get('name'), str) or not adef['name'].strip():
|
|
271
|
+
raise ActionError('An Optic action needs a name.')
|
|
272
|
+
if not isinstance(adef.get('storm'), str) or not adef['storm'].strip():
|
|
273
|
+
raise ActionError('An Optic action needs storm.')
|
|
274
|
+
forms = _strs(adef.get('forms'))
|
|
275
|
+
return {
|
|
276
|
+
'name': adef['name'],
|
|
277
|
+
'storm': adef['storm'],
|
|
278
|
+
'descr': adef.get('descr') if isinstance(adef.get('descr'), str) else None,
|
|
279
|
+
'forms': forms or ['*'],
|
|
280
|
+
'type': 'storm',
|
|
281
|
+
'group': 'Actions',
|
|
282
|
+
'subgroup': pkgname,
|
|
283
|
+
'pkg': pkgname,
|
|
284
|
+
'source': 'optic',
|
|
285
|
+
'iden': f'{pkgname}:optic:{adef["name"]}',
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
def buildRegistry(pkgdefs):
|
|
289
|
+
'''
|
|
290
|
+
The actions and note types of a set of package definitions: {actions, notetypes, errors, warnings}. errors
|
|
291
|
+
lists the entries which could not be used, warnings what was ignored or will cause trouble (unknown keys,
|
|
292
|
+
actions without an id), each {pkg, source, name, mesg}, so a package author can see why.
|
|
293
|
+
'''
|
|
294
|
+
actions = []
|
|
295
|
+
notetypes = []
|
|
296
|
+
errors = []
|
|
297
|
+
warnings = []
|
|
298
|
+
|
|
299
|
+
def note(into, pkgname, source, adef, mesg):
|
|
300
|
+
name = adef.get('name') if isinstance(adef, dict) else None
|
|
301
|
+
into.append({'pkg': pkgname, 'source': source, 'name': name if isinstance(name, str) else None, 'mesg': mesg})
|
|
302
|
+
|
|
303
|
+
def unknown(pkgname, source, adef, known):
|
|
304
|
+
extra = sorted(k for k in adef if k not in known) if isinstance(adef, dict) else ()
|
|
305
|
+
if extra:
|
|
306
|
+
note(warnings, pkgname, source, adef, f'Ignored unknown key{"s" if len(extra) > 1 else ""}: {", ".join(map(str, extra))}.')
|
|
307
|
+
|
|
308
|
+
for pkgdef in pkgdefs:
|
|
309
|
+
# a package may have neuralui actions, Optic actions, commands, all of them or none
|
|
310
|
+
info = pkgdef.get('neuralui')
|
|
311
|
+
if not isinstance(info, dict):
|
|
312
|
+
info = {}
|
|
313
|
+
|
|
314
|
+
pkgname = pkgdef.get('name')
|
|
315
|
+
if not isinstance(pkgname, str) or ':' in pkgname:
|
|
316
|
+
# an action's iden is <package>:..., so a package named a:cmd could pass for package a's actions
|
|
317
|
+
note(errors, str(pkgname), 'neuralui', {}, 'A package whose name has a colon can not have right-click actions.')
|
|
318
|
+
continue
|
|
319
|
+
commands = {c['name']: c for c in pkgdef.get('commands') or () if isinstance(c, dict) and isinstance(c.get('name'), str)}
|
|
320
|
+
|
|
321
|
+
fmtvers = info.get('version', 1)
|
|
322
|
+
if not isinstance(fmtvers, int) or isinstance(fmtvers, bool) or fmtvers < 1:
|
|
323
|
+
note(errors, pkgname, 'neuralui', {}, 'neuralui: version: must be a whole number (1).')
|
|
324
|
+
info = {}
|
|
325
|
+
elif fmtvers > FORMAT_VERSION:
|
|
326
|
+
note(errors, pkgname, 'neuralui', {},
|
|
327
|
+
f'Written for a newer NeuralUI (package format {fmtvers}; this one reads {FORMAT_VERSION}): its neuralui: actions and note types are not offered.')
|
|
328
|
+
info = {}
|
|
329
|
+
unknown(pkgname, 'neuralui', info, PKG_KEYS)
|
|
330
|
+
|
|
331
|
+
names = set()
|
|
332
|
+
ids = set()
|
|
333
|
+
curated = set()
|
|
334
|
+
for indx, adef in enumerate(info.get('actions') or ()):
|
|
335
|
+
try:
|
|
336
|
+
adef = neuralAction(adef)
|
|
337
|
+
if adef.get('id') is not None:
|
|
338
|
+
if adef['id'] in ids:
|
|
339
|
+
raise ActionError(f'Another action of the package has the id {adef["id"]!r}.')
|
|
340
|
+
ids.add(adef['id'])
|
|
341
|
+
# an action for one of the package's commands: what it leaves out comes from the command
|
|
342
|
+
if adef['type'] == 'command':
|
|
343
|
+
cdef = commands.get(adef['command'])
|
|
344
|
+
if cdef is None:
|
|
345
|
+
raise ActionError(f'The package has no command {adef["command"]!r}.')
|
|
346
|
+
curated.add(cdef['name'])
|
|
347
|
+
if not adef.get('group'):
|
|
348
|
+
adef['group'] = 'Enrich'
|
|
349
|
+
adef.setdefault('subgroup', pkgname)
|
|
350
|
+
if adef['forms'] == ['*'] and cmdForms(cdef):
|
|
351
|
+
adef['forms'] = cmdForms(cdef)
|
|
352
|
+
adef['args'] = presetFields(adef.get('args') or cmdFields(cdef.get('cmdargs')), adef.pop('values', None))
|
|
353
|
+
adef.setdefault('perms', permList(cdef.get('perms')))
|
|
354
|
+
adef.setdefault('descr', firstPara(cdef.get('descr')))
|
|
355
|
+
adef.setdefault('yields', cmdYields(cdef))
|
|
356
|
+
except ActionError as e:
|
|
357
|
+
note(errors, pkgname, 'neuralui', adef, str(e))
|
|
358
|
+
continue
|
|
359
|
+
unknown(pkgname, 'neuralui', adef, ACTION_KEYS + ('subgroup',))
|
|
360
|
+
if adef.get('id') is None:
|
|
361
|
+
note(warnings, pkgname, 'neuralui', adef,
|
|
362
|
+
'No id: it is named by its place in the list, so pins and admin settings for it move if the list is reordered.')
|
|
363
|
+
adef['pkg'] = pkgname
|
|
364
|
+
adef['source'] = 'neuralui'
|
|
365
|
+
# a stable iden when the action names one (favourites and settings survive reordering)
|
|
366
|
+
adef['iden'] = f'{pkgname}:{adef["id"]}' if adef.get('id') is not None else f'{pkgname}:{indx}'
|
|
367
|
+
names.add(adef['name'])
|
|
368
|
+
actions.append(adef)
|
|
369
|
+
|
|
370
|
+
# every command which says what forms it takes is an action on those forms (Enrich > <package>),
|
|
371
|
+
# unless the package says otherwise (commands: false, or hide: [cmd:<name>]) or one of its actions
|
|
372
|
+
# above is for that command
|
|
373
|
+
hidden = set(_strs(info.get('hide')) or ())
|
|
374
|
+
if info.get('commands') is not False:
|
|
375
|
+
for cname, cdef in commands.items():
|
|
376
|
+
forms = cmdForms(cdef)
|
|
377
|
+
if not forms or cname in curated or f'cmd:{cname}' in hidden:
|
|
378
|
+
continue
|
|
379
|
+
actions.append({
|
|
380
|
+
'name': f'{cname}…',
|
|
381
|
+
'command': cname,
|
|
382
|
+
'type': 'command',
|
|
383
|
+
'forms': forms,
|
|
384
|
+
'args': cmdFields(cdef.get('cmdargs')),
|
|
385
|
+
'perms': permList(cdef.get('perms')),
|
|
386
|
+
'descr': firstPara(cdef.get('descr')),
|
|
387
|
+
'yields': cmdYields(cdef),
|
|
388
|
+
'group': 'Enrich',
|
|
389
|
+
'subgroup': pkgname,
|
|
390
|
+
'pkg': pkgname,
|
|
391
|
+
'source': 'command',
|
|
392
|
+
'iden': f'{pkgname}:cmd:{cname}',
|
|
393
|
+
})
|
|
394
|
+
|
|
395
|
+
# Optic actions (optic: actions: in a Rapid Power-Up). A neuralui action of the same name wins.
|
|
396
|
+
optic = pkgdef.get('optic')
|
|
397
|
+
for adef in (optic.get('actions') or () if isinstance(optic, dict) else ()):
|
|
398
|
+
try:
|
|
399
|
+
act = opticAction(pkgname, adef)
|
|
400
|
+
except ActionError as e:
|
|
401
|
+
note(errors, pkgname, 'optic', adef, str(e))
|
|
402
|
+
continue
|
|
403
|
+
if act['name'] not in names:
|
|
404
|
+
actions.append(act)
|
|
405
|
+
|
|
406
|
+
for ndef in info.get('notetypes') or ():
|
|
407
|
+
if not isinstance(ndef, dict) or not isinstance(ndef.get('type'), str) or not ndef['type']:
|
|
408
|
+
note(errors, pkgname, 'notetype', ndef, 'A note type needs a type (a meta:note :type, such as acme.report).')
|
|
409
|
+
continue
|
|
410
|
+
unknown(pkgname, 'notetype', {'name': ndef['type'], **ndef}, NOTETYPE_KEYS + ('name',))
|
|
411
|
+
ndef = dict(ndef)
|
|
412
|
+
if ndef.get('render', 'markdown') not in NOTE_RENDERS:
|
|
413
|
+
note(warnings, pkgname, 'notetype', {'name': ndef['type']},
|
|
414
|
+
f'render {ndef["render"]!r} is not one of {", ".join(NOTE_RENDERS)}: shown as text.')
|
|
415
|
+
ndef['render'] = 'text'
|
|
416
|
+
ndef['pkg'] = pkgname
|
|
417
|
+
ndef.setdefault('render', 'markdown')
|
|
418
|
+
notetypes.append(ndef)
|
|
419
|
+
|
|
420
|
+
# an iden names one action (an Optic action and a neuralui action of the same package can not share one)
|
|
421
|
+
seen = set()
|
|
422
|
+
unique = []
|
|
423
|
+
for act in actions:
|
|
424
|
+
if act['iden'] in seen:
|
|
425
|
+
note(errors, act.get('pkg'), act.get('source'), act, f'Another action is already named {act["iden"]}.')
|
|
426
|
+
continue
|
|
427
|
+
seen.add(act['iden'])
|
|
428
|
+
unique.append(act)
|
|
429
|
+
|
|
430
|
+
return {'actions': unique, 'notetypes': notetypes, 'errors': errors, 'warnings': warnings}
|
|
431
|
+
|
|
432
|
+
def registryHash(pkgdefs, conf, *extra):
|
|
433
|
+
'''
|
|
434
|
+
A hash which changes when the actions could: a package added, removed or upgraded, the admin settings,
|
|
435
|
+
or anything in extra (the NeuralUI version). The page re-checks it to notice new packages.
|
|
436
|
+
'''
|
|
437
|
+
pkgs = sorted((str(p.get('name')), str(p.get('version'))) for p in pkgdefs)
|
|
438
|
+
text = json.dumps([pkgs, conf, list(extra)], sort_keys=True, default=str)
|
|
439
|
+
return hashlib.sha256(text.encode()).hexdigest()[:32]
|
|
440
|
+
|
|
441
|
+
# ---------------------------------------------------------------------- admin settings
|
|
442
|
+
|
|
443
|
+
def emptyConf():
|
|
444
|
+
return {'hidden': {}, 'pins': [], 'commands': True}
|
|
445
|
+
|
|
446
|
+
def validConf(conf):
|
|
447
|
+
'''
|
|
448
|
+
The admin's action settings, checked: {hidden: {key: {all: bool, roles: [role iden]}}, pins: [iden],
|
|
449
|
+
commands: bool}. A hidden key is an action iden, or pkg:<name> for every action of a package; commands
|
|
450
|
+
False turns off the actions made from packages' commands (they are on by default). Raises BadArg if
|
|
451
|
+
malformed.
|
|
452
|
+
'''
|
|
453
|
+
if not isinstance(conf, dict):
|
|
454
|
+
raise s_exc.BadArg(mesg='The action settings must be an object.')
|
|
455
|
+
hidden = conf.get('hidden', {}) if conf.get('hidden') is not None else {}
|
|
456
|
+
pins = conf.get('pins', []) if conf.get('pins') is not None else []
|
|
457
|
+
if not isinstance(hidden, dict) or not isinstance(pins, (list, tuple)):
|
|
458
|
+
raise s_exc.BadArg(mesg='hidden must be an object and pins a list.')
|
|
459
|
+
commands = conf.get('commands', True)
|
|
460
|
+
if not isinstance(commands, bool):
|
|
461
|
+
raise s_exc.BadArg(mesg='commands must be true or false.')
|
|
462
|
+
out = {'hidden': {}, 'pins': [], 'commands': commands}
|
|
463
|
+
for key, rule in hidden.items():
|
|
464
|
+
if not isinstance(key, str) or not key or len(key) > 512 or not isinstance(rule, dict):
|
|
465
|
+
raise s_exc.BadArg(mesg='Each hidden entry maps an action (or pkg:<name>) to {all, roles}.')
|
|
466
|
+
roles = rule.get('roles') or []
|
|
467
|
+
if not isinstance(roles, (list, tuple)) or not all(isinstance(r, str) for r in roles):
|
|
468
|
+
raise s_exc.BadArg(mesg='roles must be a list of role idens.')
|
|
469
|
+
rule = {'all': rule.get('all') is True, 'roles': sorted(set(roles))}
|
|
470
|
+
if rule['all'] or rule['roles']:
|
|
471
|
+
out['hidden'][key] = rule
|
|
472
|
+
for iden in pins:
|
|
473
|
+
if not isinstance(iden, str) or not iden or len(iden) > 512:
|
|
474
|
+
raise s_exc.BadArg(mesg='pins must be a list of action idens.')
|
|
475
|
+
if iden not in out['pins']:
|
|
476
|
+
out['pins'].append(iden)
|
|
477
|
+
if len(out['hidden']) > 10000 or len(out['pins']) > 200:
|
|
478
|
+
raise s_exc.BadArg(mesg='Too many action settings.')
|
|
479
|
+
return out
|
|
480
|
+
|
|
481
|
+
def renameIdens(conf, aliases):
|
|
482
|
+
'''The settings with action idens renamed by aliases ({old iden: new}): hidden keys and pins.'''
|
|
483
|
+
hidden = {}
|
|
484
|
+
for key, rule in conf['hidden'].items():
|
|
485
|
+
hidden.setdefault(aliases.get(key, key), rule)
|
|
486
|
+
pins = []
|
|
487
|
+
for iden in conf['pins']:
|
|
488
|
+
iden = aliases.get(iden, iden)
|
|
489
|
+
if iden not in pins:
|
|
490
|
+
pins.append(iden)
|
|
491
|
+
return {**conf, 'hidden': hidden, 'pins': pins}
|
|
492
|
+
|
|
493
|
+
def isHidden(act, conf, roles):
|
|
494
|
+
'''True if the admin hid the action (or its package) from everyone, or from one of the user's roles.'''
|
|
495
|
+
for key in (act.get('iden'), f'pkg:{act.get("pkg")}'):
|
|
496
|
+
rule = conf['hidden'].get(key)
|
|
497
|
+
if rule and (rule['all'] or set(rule['roles']) & set(roles)):
|
|
498
|
+
return True
|
|
499
|
+
return False
|
|
500
|
+
|
|
501
|
+
def visibleActions(actions, conf, roles):
|
|
502
|
+
'''The actions a user with these role idens sees, as the admin set them.'''
|
|
503
|
+
offcmds = conf.get('commands') is False
|
|
504
|
+
return [a for a in actions if not isHidden(a, conf, roles) and not (offcmds and a.get('source') == 'command')]
|
neuralui/checkpkg.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Check a Storm package's neuralui: key, as a package author would before releasing it.
|
|
3
|
+
|
|
4
|
+
python -m neuralui.checkpkg path/to/pkgproto.yaml [more.yaml|pkgdef.json ...]
|
|
5
|
+
|
|
6
|
+
Each path is a package prototype (YAML, loaded as Synapse's genpkg loads it) or a built package definition
|
|
7
|
+
(JSON). For each, it checks the neuralui: key against the JSON Schema (neuralui/schema/neuralui.schema.json),
|
|
8
|
+
then builds the right-click actions as NeuralUI would and lists what it could not use and its warnings.
|
|
9
|
+
Exits 1 if anything is wrong, 0 if not.
|
|
10
|
+
'''
|
|
11
|
+
import sys
|
|
12
|
+
import json
|
|
13
|
+
import argparse
|
|
14
|
+
|
|
15
|
+
import synapse.tools.storm.pkg.gen as s_genpkg
|
|
16
|
+
|
|
17
|
+
import neuralui.actions as w_actions
|
|
18
|
+
|
|
19
|
+
def loadPkg(path):
|
|
20
|
+
if path.endswith('.json'):
|
|
21
|
+
with open(path) as fd:
|
|
22
|
+
return json.load(fd)
|
|
23
|
+
return json.loads(json.dumps(s_genpkg.tryLoadPkgProto(path, readonly=True)))
|
|
24
|
+
|
|
25
|
+
def check(pkgdef):
|
|
26
|
+
'''The problems with a package definition's neuralui: key, as lines of text (none when it is fine).'''
|
|
27
|
+
out = []
|
|
28
|
+
info = pkgdef.get('neuralui')
|
|
29
|
+
if info is not None:
|
|
30
|
+
mesg = w_actions.checkSchema(info)
|
|
31
|
+
if mesg:
|
|
32
|
+
out.append(f'schema: {mesg}')
|
|
33
|
+
reg = w_actions.buildRegistry([pkgdef])
|
|
34
|
+
for kind in ('errors', 'warnings'):
|
|
35
|
+
for item in reg[kind]:
|
|
36
|
+
out.append(f'{kind[:-1]}: {item["name"] or "(the package)"} ({item["source"]}): {item["mesg"]}')
|
|
37
|
+
return out
|
|
38
|
+
|
|
39
|
+
def main(argv):
|
|
40
|
+
pars = argparse.ArgumentParser(prog='python -m neuralui.checkpkg', description=__doc__.strip().split('\n')[0])
|
|
41
|
+
pars.add_argument('paths', nargs='+', help='Package prototypes (.yaml) or package definitions (.json).')
|
|
42
|
+
opts = pars.parse_args(argv)
|
|
43
|
+
bad = 0
|
|
44
|
+
for path in opts.paths:
|
|
45
|
+
try:
|
|
46
|
+
pkgdef = loadPkg(path)
|
|
47
|
+
except Exception as e:
|
|
48
|
+
print(f'{path}: could not load: {e}')
|
|
49
|
+
bad += 1
|
|
50
|
+
continue
|
|
51
|
+
lines = check(pkgdef)
|
|
52
|
+
acts = len(w_actions.buildRegistry([pkgdef])['actions'])
|
|
53
|
+
print(f'{path}: {pkgdef.get("name")}: {acts} action{"" if acts == 1 else "s"}' + ('' if lines else ', no problems'))
|
|
54
|
+
for line in lines:
|
|
55
|
+
print(f' {line}')
|
|
56
|
+
bad += bool(lines)
|
|
57
|
+
return 1 if bad else 0
|
|
58
|
+
|
|
59
|
+
if __name__ == '__main__': # pragma: no cover
|
|
60
|
+
sys.exit(main(sys.argv[1:]))
|