neuralui 0.12.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. neuralui/__init__.py +0 -0
  2. neuralui/__main__.py +7 -0
  3. neuralui/actions.py +504 -0
  4. neuralui/checkpkg.py +60 -0
  5. neuralui/export.py +103 -0
  6. neuralui/handlers.py +1885 -0
  7. neuralui/jobs.py +400 -0
  8. neuralui/limits.py +71 -0
  9. neuralui/lint.py +508 -0
  10. neuralui/pkg.py +27 -0
  11. neuralui/pkgproto/neuralui.yaml +195 -0
  12. neuralui/pkgproto/storm/commands/neuralui.query.migrate.storm +12 -0
  13. neuralui/pkgproto/storm/commands/neuralui.query.save.storm +1 -0
  14. neuralui/pkgproto/storm/commands/neuralui.scrape.migrate.storm +20 -0
  15. neuralui/pkgproto/storm/modules/neuralui.storm +121 -0
  16. neuralui/schema/neuralui.schema.json +144 -0
  17. neuralui/security.py +265 -0
  18. neuralui/service.py +342 -0
  19. neuralui/store.py +399 -0
  20. neuralui/version.py +12 -0
  21. neuralui/web/brand/neuralui-compass-primary-180.png +0 -0
  22. neuralui/web/brand/neuralui-compass-primary-light.svg +1 -0
  23. neuralui/web/brand/neuralui-compass-primary.svg +1 -0
  24. neuralui/web/brand/neuralui-compass-small-32.png +0 -0
  25. neuralui/web/brand/neuralui-compass-small-light.svg +1 -0
  26. neuralui/web/brand/neuralui-compass-small.svg +1 -0
  27. neuralui/web/css/app.css +1420 -0
  28. neuralui/web/docs/actions.md +137 -0
  29. neuralui/web/docs/admin.md +45 -0
  30. neuralui/web/docs/automation.md +48 -0
  31. neuralui/web/docs/console.md +25 -0
  32. neuralui/web/docs/details.md +55 -0
  33. neuralui/web/docs/explore.md +11 -0
  34. neuralui/web/docs/graph.md +71 -0
  35. neuralui/web/docs/history.md +83 -0
  36. neuralui/web/docs/img/about.png +0 -0
  37. neuralui/web/docs/img/action-palette.png +0 -0
  38. neuralui/web/docs/img/add-page.png +0 -0
  39. neuralui/web/docs/img/admin-actions.png +0 -0
  40. neuralui/web/docs/img/applicator.png +0 -0
  41. neuralui/web/docs/img/automation.png +0 -0
  42. neuralui/web/docs/img/bulkedit.png +0 -0
  43. neuralui/web/docs/img/command-dialog.png +0 -0
  44. neuralui/web/docs/img/completion.png +0 -0
  45. neuralui/web/docs/img/drill.png +0 -0
  46. neuralui/web/docs/img/enrich-menu.png +0 -0
  47. neuralui/web/docs/img/error-state.png +0 -0
  48. neuralui/web/docs/img/graph-large.png +0 -0
  49. neuralui/web/docs/img/graph-okabe-ito.png +0 -0
  50. neuralui/web/docs/img/graph.png +0 -0
  51. neuralui/web/docs/img/guide.png +0 -0
  52. neuralui/web/docs/img/history.png +0 -0
  53. neuralui/web/docs/img/import.png +0 -0
  54. neuralui/web/docs/img/jobs-panel.png +0 -0
  55. neuralui/web/docs/img/lint.png +0 -0
  56. neuralui/web/docs/img/menu.png +0 -0
  57. neuralui/web/docs/img/note.png +0 -0
  58. neuralui/web/docs/img/open-in.png +0 -0
  59. neuralui/web/docs/img/output-view.png +0 -0
  60. neuralui/web/docs/img/palette.png +0 -0
  61. neuralui/web/docs/img/scrape-results.png +0 -0
  62. neuralui/web/docs/img/scrape-review.png +0 -0
  63. neuralui/web/docs/img/script.png +0 -0
  64. neuralui/web/docs/img/table.png +0 -0
  65. neuralui/web/docs/img/tagnode.png +0 -0
  66. neuralui/web/docs/img/tags.png +0 -0
  67. neuralui/web/docs/img/theme-high-contrast.png +0 -0
  68. neuralui/web/docs/img/theme-light.png +0 -0
  69. neuralui/web/docs/img/update-banner.png +0 -0
  70. neuralui/web/docs/img/viewer-hex.png +0 -0
  71. neuralui/web/docs/import.md +57 -0
  72. neuralui/web/docs/keyboard.md +45 -0
  73. neuralui/web/docs/library.md +49 -0
  74. neuralui/web/docs/overview.md +76 -0
  75. neuralui/web/docs/queries.md +103 -0
  76. neuralui/web/docs/results.md +103 -0
  77. neuralui/web/docs/scrape.md +39 -0
  78. neuralui/web/docs/settings.md +94 -0
  79. neuralui/web/docs/tags.md +88 -0
  80. neuralui/web/index.html +21 -0
  81. neuralui/web/js/actionpalette.js +99 -0
  82. neuralui/web/js/actions.js +165 -0
  83. neuralui/web/js/addpage.js +62 -0
  84. neuralui/web/js/admin.js +548 -0
  85. neuralui/web/js/api.js +316 -0
  86. neuralui/web/js/app.js +529 -0
  87. neuralui/web/js/automation.js +723 -0
  88. neuralui/web/js/boot.js +7 -0
  89. neuralui/web/js/complete.js +272 -0
  90. neuralui/web/js/console.js +204 -0
  91. neuralui/web/js/dialog.js +267 -0
  92. neuralui/web/js/drill.js +301 -0
  93. neuralui/web/js/editor.js +725 -0
  94. neuralui/web/js/editors.js +640 -0
  95. neuralui/web/js/explore.js +161 -0
  96. neuralui/web/js/export.js +222 -0
  97. neuralui/web/js/files.js +85 -0
  98. neuralui/web/js/graph.js +530 -0
  99. neuralui/web/js/help.js +35 -0
  100. neuralui/web/js/helppage.js +304 -0
  101. neuralui/web/js/import.js +408 -0
  102. neuralui/web/js/importfiles.js +262 -0
  103. neuralui/web/js/jobs.js +211 -0
  104. neuralui/web/js/layout-worker.js +107 -0
  105. neuralui/web/js/livemd.js +509 -0
  106. neuralui/web/js/lookups.js +81 -0
  107. neuralui/web/js/menu.js +752 -0
  108. neuralui/web/js/model.js +267 -0
  109. neuralui/web/js/notepage.js +111 -0
  110. neuralui/web/js/notes.js +363 -0
  111. neuralui/web/js/page.js +91 -0
  112. neuralui/web/js/palette.js +128 -0
  113. neuralui/web/js/queries.js +163 -0
  114. neuralui/web/js/results.js +1006 -0
  115. neuralui/web/js/saved.js +250 -0
  116. neuralui/web/js/scrape.js +328 -0
  117. neuralui/web/js/script.js +373 -0
  118. neuralui/web/js/state.js +268 -0
  119. neuralui/web/js/tabs.js +1118 -0
  120. neuralui/web/js/tags.js +114 -0
  121. neuralui/web/js/undo.js +104 -0
  122. neuralui/web/js/util.js +195 -0
  123. neuralui/web/js/viewer.js +437 -0
  124. neuralui/web/vendor/cytoscape.LICENSE +19 -0
  125. neuralui/web/vendor/cytoscape.min.js +32 -0
  126. neuralui/web/vendor/dompurify.LICENSE +202 -0
  127. neuralui/web/vendor/inter/OFL.txt +93 -0
  128. neuralui/web/vendor/inter/inter-latin-ext-wght-normal.woff2 +0 -0
  129. neuralui/web/vendor/inter/inter-latin-wght-normal.woff2 +0 -0
  130. neuralui/web/vendor/marked.LICENSE +44 -0
  131. neuralui/web/vendor/marked.min.js +6 -0
  132. neuralui/web/vendor/purify.min.js +7 -0
  133. neuralui-0.12.2.dist-info/METADATA +679 -0
  134. neuralui-0.12.2.dist-info/RECORD +136 -0
  135. neuralui-0.12.2.dist-info/WHEEL +5 -0
  136. neuralui-0.12.2.dist-info/top_level.txt +1 -0
neuralui/__init__.py ADDED
File without changes
neuralui/__main__.py ADDED
@@ -0,0 +1,7 @@
1
+ import sys
2
+ import asyncio
3
+
4
+ import neuralui.service as w_service
5
+
6
+ if __name__ == '__main__': # pragma: no cover
7
+ asyncio.run(w_service.NeuralUI.execmain(sys.argv[1:]))
neuralui/actions.py ADDED
@@ -0,0 +1,504 @@
1
+ '''
2
+ The right-click action registry: every node action the UI offers, built from the Storm packages in the Cortex.
3
+
4
+ Actions come from four places (docs/PACKAGES.md describes the formats, neuralui/schema/neuralui.schema.json
5
+ is the JSON Schema of the neuralui: key):
6
+
7
+ neuralui a package's neuralui: actions: (query, newtab, filter, storm, link and command actions)
8
+ optic a Rapid Power-Up's optic: actions:, shown as Optic shows them (Actions > <package>)
9
+ command every package command which says what forms it takes (Enrich > <package>)
10
+ defaults the neuralui package's own actions (pivots, filters, Open in ...), even before service.add
11
+
12
+ buildRegistry() turns package definitions into the list of actions (with the entries it could not use, and
13
+ why, for the Admin page). An admin's settings (actions hidden for everyone or for roles, default pins) are
14
+ applied per user by visibleActions(); the user's permissions are checked by the HTTP handler.
15
+ '''
16
+ import os
17
+ import re
18
+ import json
19
+ import hashlib
20
+ import functools
21
+
22
+ import synapse.exc as s_exc
23
+ import synapse.lib.config as s_config
24
+
25
+ # a link action's templates must name a web page: no javascript:, data: or other schemes
26
+ LINK_PREFIXES = ('https://', 'http://')
27
+
28
+ ACTION_TYPES = ('query', 'newtab', 'filter', 'storm', 'link', 'command')
29
+
30
+ # menu: where an action goes at the top of the menu (its group, if it has one, becomes the submenu inside)
31
+ MENUS = {
32
+ 'pivot': 'Pivot',
33
+ 'filter': 'Filter',
34
+ 'enrich': 'Enrich',
35
+ 'edit': 'Edit',
36
+ 'open': 'Open in',
37
+ 'actions': 'Actions',
38
+ }
39
+
40
+ YIELDS = ('inbound', 'new', 'none')
41
+
42
+ # the version of the neuralui: package format this NeuralUI reads (a package's neuralui: version:, default 1).
43
+ # A newer minor NeuralUI may add optional keys; a package which needs a newer format says so with version:.
44
+ FORMAT_VERSION = 1
45
+
46
+ # the keys NeuralUI reads. Others are ignored (so a package written for a newer 1.x still loads) and listed
47
+ # as warnings in Admin > Actions, where a typo shows up.
48
+ PKG_KEYS = ('version', 'actions', 'notetypes', 'hide', 'commands')
49
+ ACTION_KEYS = ('id', 'name', 'type', 'forms', 'group', 'menu', 'storm', 'url', 'command', 'values', 'args',
50
+ 'descr', 'multi', 'confirm', 'perms', 'max', 'batch', 'yields', 'background')
51
+ NOTETYPE_KEYS = ('type', 'title', 'render', 'descr')
52
+ NOTE_RENDERS = ('markdown', 'text')
53
+
54
+ # an action id: stable within its package, and part of its iden (<pkg>:<id>), so no colons
55
+ ID_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$')
56
+
57
+ # the most nodes, and batch size, a storm or command action may set (the page defaults to 500 and 100)
58
+ MAX_MAX = 100000
59
+
60
+ SCHEMA_PATH = os.path.join(os.path.dirname(__file__), 'schema', 'neuralui.schema.json')
61
+
62
+ @functools.cache
63
+ def _validator():
64
+ with open(SCHEMA_PATH) as fd:
65
+ return s_config.getJsValidator(json.load(fd))
66
+
67
+ def checkSchema(info):
68
+ '''
69
+ Check a package's neuralui: key against the JSON Schema (stricter than NeuralUI, which ignores unknown
70
+ keys). Returns the problem as a message, or None when it is valid.
71
+ '''
72
+ try:
73
+ _validator()(json.loads(json.dumps(info)))
74
+ except s_exc.SchemaViolation as e:
75
+ return e.get('mesg') or str(e)
76
+ return None
77
+
78
+ class ActionError(Exception):
79
+ '''An action entry which can not be used: the message says why (shown in Admin > Actions).'''
80
+
81
+ def _strs(valu):
82
+ '''A list of strings from a list or tuple (packages read back from the Cortex have tuples), or None.'''
83
+ if not isinstance(valu, (list, tuple)):
84
+ return None
85
+ return [v for v in valu if isinstance(v, str)]
86
+
87
+ def _posint(valu, most=MAX_MAX):
88
+ return valu if isinstance(valu, int) and not isinstance(valu, bool) and 0 < valu <= most else None
89
+
90
+ def neuralAction(adef):
91
+ '''
92
+ A package's neuralui action, checked, with its defaults. Raises ActionError when it can not be used.
93
+
94
+ A link action (type: link) opens a web page for a node instead of running Storm: its url is a template,
95
+ or a {form: template} map, or a {form: [template, ...]} map tried in order (see docs/PACKAGES.md).
96
+ '''
97
+ if not isinstance(adef, dict):
98
+ raise ActionError('An action must be an object.')
99
+ if not isinstance(adef.get('name'), str) or not adef['name'].strip():
100
+ raise ActionError('An action needs a name.')
101
+ adef = dict(adef)
102
+
103
+ forms = _strs(adef.get('forms', ['*']))
104
+ if not forms:
105
+ raise ActionError('forms must be a list of form names or globs.')
106
+ adef['forms'] = forms
107
+
108
+ if adef.get('id') is not None and (not isinstance(adef['id'], str) or not ID_RE.match(adef['id'])):
109
+ raise ActionError('id must be letters, digits, dots, dashes and underscores (up to 128), starting with a letter or digit.')
110
+
111
+ kind = adef.setdefault('type', 'command' if isinstance(adef.get('command'), str) else 'query')
112
+ if kind not in ACTION_TYPES:
113
+ raise ActionError(f'Unknown type {kind!r} (one of {", ".join(ACTION_TYPES)}).')
114
+
115
+ # where it goes in the menu: menu picks the top level, and a group becomes the submenu inside it
116
+ menu = adef.pop('menu', None)
117
+ if menu is not None:
118
+ if menu not in MENUS:
119
+ raise ActionError(f'Unknown menu {menu!r} (one of {", ".join(MENUS)}).')
120
+ if adef.get('group') and not adef.get('subgroup'):
121
+ adef['subgroup'] = adef['group']
122
+ adef['group'] = MENUS[menu]
123
+
124
+ for name in ('max', 'batch'):
125
+ if adef.get(name) is not None:
126
+ if _posint(adef[name]) is None:
127
+ raise ActionError(f'{name} must be a whole number from 1 to {MAX_MAX}.')
128
+ if adef.get('yields') is not None and adef['yields'] not in YIELDS:
129
+ raise ActionError(f'yields must be one of {", ".join(YIELDS)}.')
130
+ if adef.get('perms') is not None:
131
+ perms = permList(adef['perms'])
132
+ if perms is None:
133
+ raise ActionError('perms must be a list of permission paths, such as [["acme", "user"]].')
134
+ adef['perms'] = perms
135
+
136
+ if kind == 'link':
137
+ url = adef.get('url')
138
+ if isinstance(url, str):
139
+ url = {'*': [url]}
140
+ if not isinstance(url, dict) or not url:
141
+ raise ActionError('A link action needs a url: a template, or a map of form to templates.')
142
+ urls = {}
143
+ for form, temps in url.items():
144
+ temps = [temps] if isinstance(temps, str) else temps
145
+ if not isinstance(form, str) or not isinstance(temps, (list, tuple)) or not temps:
146
+ raise ActionError('A link url maps a form to a template or a list of templates.')
147
+ for temp in temps:
148
+ if not isinstance(temp, str) or not temp.startswith(LINK_PREFIXES):
149
+ raise ActionError(f'A link template must start with https:// or http:// ({temp!r} does not).')
150
+ urls[form] = list(temps)
151
+ adef['url'] = urls
152
+ if adef['forms'] == ['*']:
153
+ adef['forms'] = list(urls)
154
+ # a link opens one browser tab: it is for one node at a time
155
+ adef['multi'] = False
156
+ return adef
157
+
158
+ if kind == 'command':
159
+ # runs one of the package's commands (its args, forms and perms come from the command)
160
+ if not isinstance(adef.get('command'), str) or not adef['command']:
161
+ raise ActionError('A command action needs the name of one of the package\'s commands.')
162
+ if adef.get('values') is not None and not isinstance(adef['values'], dict):
163
+ raise ActionError('values must map argument names to values.')
164
+ return adef
165
+
166
+ if not isinstance(adef.get('storm'), str) or not adef['storm'].strip():
167
+ raise ActionError(f'A {kind} action needs storm.')
168
+ return adef
169
+
170
+ def cmdForms(cdef):
171
+ '''The forms a Storm command takes as input: cmdinputs, or the older forms: input: hints.'''
172
+ forms = []
173
+ for item in cdef.get('cmdinputs') or ():
174
+ if isinstance(item, dict) and isinstance(item.get('form'), str):
175
+ forms.append(item['form'])
176
+ hints = cdef.get('forms')
177
+ if not forms and isinstance(hints, dict):
178
+ forms = [f for f in hints.get('input') or () if isinstance(f, str)]
179
+ return forms
180
+
181
+ def cmdYields(cdef):
182
+ '''
183
+ What a command's output is for the results: 'new' when it says it yields forms it does not take (show
184
+ them in a new tab), else 'inbound' (the inbound nodes, updated in place).
185
+ '''
186
+ hints = cdef.get('forms')
187
+ output = set(_strs(hints.get('output')) or ()) if isinstance(hints, dict) else set()
188
+ return 'new' if output - set(cmdForms(cdef)) else 'inbound'
189
+
190
+ def cmdFields(cmdargs):
191
+ '''
192
+ A Storm command's arguments (cmdargs, argparse style) as prompt fields. Each field keeps how it goes on
193
+ the command line: flag (None for a positional argument), and for a list whether the flag repeats per
194
+ value (append) or takes them all (nargs). A store_true / store_false flag becomes a checkbox.
195
+ '''
196
+ fields = []
197
+ for item in cmdargs or ():
198
+ if not isinstance(item, (list, tuple)) or not item or not isinstance(item[0], str):
199
+ continue
200
+ name = item[0]
201
+ opts = item[1] if len(item) > 1 and isinstance(item[1], dict) else {}
202
+ if name in ('-h', '--help'):
203
+ continue
204
+ positional = not name.startswith('-')
205
+ action = opts.get('action')
206
+ nargs = opts.get('nargs')
207
+ helptext = opts.get('help') if isinstance(opts.get('help'), str) else ''
208
+ field = {
209
+ 'name': opts.get('dest') if isinstance(opts.get('dest'), str) else name.lstrip('-').replace('-', '_'),
210
+ 'label': name,
211
+ 'flag': None if positional else name,
212
+ 'help': helptext.strip(),
213
+ }
214
+ default = opts.get('default')
215
+ if action in ('store_true', 'store_false'):
216
+ field['type'] = 'bool'
217
+ field['action'] = action
218
+ # the box shows what the command does when the flag is left out
219
+ field['default'] = action == 'store_false'
220
+ elif action == 'append' or nargs in ('*', '+') or (isinstance(nargs, int) and nargs > 1):
221
+ field['type'] = 'list'
222
+ field['mode'] = 'append' if action == 'append' else 'nargs'
223
+ elif isinstance(opts.get('choices'), (list, tuple)) and opts['choices']:
224
+ field['type'] = 'select'
225
+ choices = [c for c in opts['choices'] if isinstance(c, (str, int, float))]
226
+ field['options'] = ([] if positional else [['', '(default)']]) + [[str(c), str(c)] for c in choices]
227
+ elif opts.get('type') == 'int':
228
+ field['type'] = 'int'
229
+ else:
230
+ field['type'] = 'str'
231
+ if field['type'] != 'bool' and isinstance(default, (str, int, float)) and not isinstance(default, bool):
232
+ field['default'] = default
233
+ if field['type'] == 'list' and isinstance(default, (list, tuple)):
234
+ field['default'] = [str(d) for d in default]
235
+ if (positional and nargs not in ('?', '*')) or opts.get('required') is True:
236
+ field['required'] = True
237
+ fields.append(field)
238
+ return fields
239
+
240
+ def presetFields(fields, values):
241
+ '''A command action's fields with its preset values as their defaults (the dialog still shows them).'''
242
+ if not values:
243
+ return fields
244
+ out = []
245
+ for field in fields:
246
+ if field['name'] in values:
247
+ field = dict(field)
248
+ valu = values[field['name']]
249
+ if field['type'] == 'list' and isinstance(valu, (list, tuple)):
250
+ valu = [str(v) for v in valu]
251
+ field['default'] = valu
252
+ out.append(field)
253
+ return out
254
+
255
+ def firstPara(text):
256
+ '''The first paragraph of a description, for a tooltip.'''
257
+ if not isinstance(text, str):
258
+ return None
259
+ return text.strip().split('\n\n', 1)[0].strip() or None
260
+
261
+ def permList(perms):
262
+ '''A command's perms as a list of perm paths (lists of names), or None.'''
263
+ if not isinstance(perms, (list, tuple)):
264
+ return None
265
+ perms = [list(p) for p in perms if isinstance(p, (list, tuple)) and p and all(isinstance(n, str) for n in p)]
266
+ return perms or None
267
+
268
+ def opticAction(pkgname, adef):
269
+ '''An Optic action (optic: actions:) as Optic shows it: Actions > <package> > <name>, run as storm.'''
270
+ if not isinstance(adef, dict) or not isinstance(adef.get('name'), str) or not adef['name'].strip():
271
+ raise ActionError('An Optic action needs a name.')
272
+ if not isinstance(adef.get('storm'), str) or not adef['storm'].strip():
273
+ raise ActionError('An Optic action needs storm.')
274
+ forms = _strs(adef.get('forms'))
275
+ return {
276
+ 'name': adef['name'],
277
+ 'storm': adef['storm'],
278
+ 'descr': adef.get('descr') if isinstance(adef.get('descr'), str) else None,
279
+ 'forms': forms or ['*'],
280
+ 'type': 'storm',
281
+ 'group': 'Actions',
282
+ 'subgroup': pkgname,
283
+ 'pkg': pkgname,
284
+ 'source': 'optic',
285
+ 'iden': f'{pkgname}:optic:{adef["name"]}',
286
+ }
287
+
288
+ def buildRegistry(pkgdefs):
289
+ '''
290
+ The actions and note types of a set of package definitions: {actions, notetypes, errors, warnings}. errors
291
+ lists the entries which could not be used, warnings what was ignored or will cause trouble (unknown keys,
292
+ actions without an id), each {pkg, source, name, mesg}, so a package author can see why.
293
+ '''
294
+ actions = []
295
+ notetypes = []
296
+ errors = []
297
+ warnings = []
298
+
299
+ def note(into, pkgname, source, adef, mesg):
300
+ name = adef.get('name') if isinstance(adef, dict) else None
301
+ into.append({'pkg': pkgname, 'source': source, 'name': name if isinstance(name, str) else None, 'mesg': mesg})
302
+
303
+ def unknown(pkgname, source, adef, known):
304
+ extra = sorted(k for k in adef if k not in known) if isinstance(adef, dict) else ()
305
+ if extra:
306
+ note(warnings, pkgname, source, adef, f'Ignored unknown key{"s" if len(extra) > 1 else ""}: {", ".join(map(str, extra))}.')
307
+
308
+ for pkgdef in pkgdefs:
309
+ # a package may have neuralui actions, Optic actions, commands, all of them or none
310
+ info = pkgdef.get('neuralui')
311
+ if not isinstance(info, dict):
312
+ info = {}
313
+
314
+ pkgname = pkgdef.get('name')
315
+ if not isinstance(pkgname, str) or ':' in pkgname:
316
+ # an action's iden is <package>:..., so a package named a:cmd could pass for package a's actions
317
+ note(errors, str(pkgname), 'neuralui', {}, 'A package whose name has a colon can not have right-click actions.')
318
+ continue
319
+ commands = {c['name']: c for c in pkgdef.get('commands') or () if isinstance(c, dict) and isinstance(c.get('name'), str)}
320
+
321
+ fmtvers = info.get('version', 1)
322
+ if not isinstance(fmtvers, int) or isinstance(fmtvers, bool) or fmtvers < 1:
323
+ note(errors, pkgname, 'neuralui', {}, 'neuralui: version: must be a whole number (1).')
324
+ info = {}
325
+ elif fmtvers > FORMAT_VERSION:
326
+ note(errors, pkgname, 'neuralui', {},
327
+ f'Written for a newer NeuralUI (package format {fmtvers}; this one reads {FORMAT_VERSION}): its neuralui: actions and note types are not offered.')
328
+ info = {}
329
+ unknown(pkgname, 'neuralui', info, PKG_KEYS)
330
+
331
+ names = set()
332
+ ids = set()
333
+ curated = set()
334
+ for indx, adef in enumerate(info.get('actions') or ()):
335
+ try:
336
+ adef = neuralAction(adef)
337
+ if adef.get('id') is not None:
338
+ if adef['id'] in ids:
339
+ raise ActionError(f'Another action of the package has the id {adef["id"]!r}.')
340
+ ids.add(adef['id'])
341
+ # an action for one of the package's commands: what it leaves out comes from the command
342
+ if adef['type'] == 'command':
343
+ cdef = commands.get(adef['command'])
344
+ if cdef is None:
345
+ raise ActionError(f'The package has no command {adef["command"]!r}.')
346
+ curated.add(cdef['name'])
347
+ if not adef.get('group'):
348
+ adef['group'] = 'Enrich'
349
+ adef.setdefault('subgroup', pkgname)
350
+ if adef['forms'] == ['*'] and cmdForms(cdef):
351
+ adef['forms'] = cmdForms(cdef)
352
+ adef['args'] = presetFields(adef.get('args') or cmdFields(cdef.get('cmdargs')), adef.pop('values', None))
353
+ adef.setdefault('perms', permList(cdef.get('perms')))
354
+ adef.setdefault('descr', firstPara(cdef.get('descr')))
355
+ adef.setdefault('yields', cmdYields(cdef))
356
+ except ActionError as e:
357
+ note(errors, pkgname, 'neuralui', adef, str(e))
358
+ continue
359
+ unknown(pkgname, 'neuralui', adef, ACTION_KEYS + ('subgroup',))
360
+ if adef.get('id') is None:
361
+ note(warnings, pkgname, 'neuralui', adef,
362
+ 'No id: it is named by its place in the list, so pins and admin settings for it move if the list is reordered.')
363
+ adef['pkg'] = pkgname
364
+ adef['source'] = 'neuralui'
365
+ # a stable iden when the action names one (favourites and settings survive reordering)
366
+ adef['iden'] = f'{pkgname}:{adef["id"]}' if adef.get('id') is not None else f'{pkgname}:{indx}'
367
+ names.add(adef['name'])
368
+ actions.append(adef)
369
+
370
+ # every command which says what forms it takes is an action on those forms (Enrich > <package>),
371
+ # unless the package says otherwise (commands: false, or hide: [cmd:<name>]) or one of its actions
372
+ # above is for that command
373
+ hidden = set(_strs(info.get('hide')) or ())
374
+ if info.get('commands') is not False:
375
+ for cname, cdef in commands.items():
376
+ forms = cmdForms(cdef)
377
+ if not forms or cname in curated or f'cmd:{cname}' in hidden:
378
+ continue
379
+ actions.append({
380
+ 'name': f'{cname}…',
381
+ 'command': cname,
382
+ 'type': 'command',
383
+ 'forms': forms,
384
+ 'args': cmdFields(cdef.get('cmdargs')),
385
+ 'perms': permList(cdef.get('perms')),
386
+ 'descr': firstPara(cdef.get('descr')),
387
+ 'yields': cmdYields(cdef),
388
+ 'group': 'Enrich',
389
+ 'subgroup': pkgname,
390
+ 'pkg': pkgname,
391
+ 'source': 'command',
392
+ 'iden': f'{pkgname}:cmd:{cname}',
393
+ })
394
+
395
+ # Optic actions (optic: actions: in a Rapid Power-Up). A neuralui action of the same name wins.
396
+ optic = pkgdef.get('optic')
397
+ for adef in (optic.get('actions') or () if isinstance(optic, dict) else ()):
398
+ try:
399
+ act = opticAction(pkgname, adef)
400
+ except ActionError as e:
401
+ note(errors, pkgname, 'optic', adef, str(e))
402
+ continue
403
+ if act['name'] not in names:
404
+ actions.append(act)
405
+
406
+ for ndef in info.get('notetypes') or ():
407
+ if not isinstance(ndef, dict) or not isinstance(ndef.get('type'), str) or not ndef['type']:
408
+ note(errors, pkgname, 'notetype', ndef, 'A note type needs a type (a meta:note :type, such as acme.report).')
409
+ continue
410
+ unknown(pkgname, 'notetype', {'name': ndef['type'], **ndef}, NOTETYPE_KEYS + ('name',))
411
+ ndef = dict(ndef)
412
+ if ndef.get('render', 'markdown') not in NOTE_RENDERS:
413
+ note(warnings, pkgname, 'notetype', {'name': ndef['type']},
414
+ f'render {ndef["render"]!r} is not one of {", ".join(NOTE_RENDERS)}: shown as text.')
415
+ ndef['render'] = 'text'
416
+ ndef['pkg'] = pkgname
417
+ ndef.setdefault('render', 'markdown')
418
+ notetypes.append(ndef)
419
+
420
+ # an iden names one action (an Optic action and a neuralui action of the same package can not share one)
421
+ seen = set()
422
+ unique = []
423
+ for act in actions:
424
+ if act['iden'] in seen:
425
+ note(errors, act.get('pkg'), act.get('source'), act, f'Another action is already named {act["iden"]}.')
426
+ continue
427
+ seen.add(act['iden'])
428
+ unique.append(act)
429
+
430
+ return {'actions': unique, 'notetypes': notetypes, 'errors': errors, 'warnings': warnings}
431
+
432
+ def registryHash(pkgdefs, conf, *extra):
433
+ '''
434
+ A hash which changes when the actions could: a package added, removed or upgraded, the admin settings,
435
+ or anything in extra (the NeuralUI version). The page re-checks it to notice new packages.
436
+ '''
437
+ pkgs = sorted((str(p.get('name')), str(p.get('version'))) for p in pkgdefs)
438
+ text = json.dumps([pkgs, conf, list(extra)], sort_keys=True, default=str)
439
+ return hashlib.sha256(text.encode()).hexdigest()[:32]
440
+
441
+ # ---------------------------------------------------------------------- admin settings
442
+
443
+ def emptyConf():
444
+ return {'hidden': {}, 'pins': [], 'commands': True}
445
+
446
+ def validConf(conf):
447
+ '''
448
+ The admin's action settings, checked: {hidden: {key: {all: bool, roles: [role iden]}}, pins: [iden],
449
+ commands: bool}. A hidden key is an action iden, or pkg:<name> for every action of a package; commands
450
+ False turns off the actions made from packages' commands (they are on by default). Raises BadArg if
451
+ malformed.
452
+ '''
453
+ if not isinstance(conf, dict):
454
+ raise s_exc.BadArg(mesg='The action settings must be an object.')
455
+ hidden = conf.get('hidden', {}) if conf.get('hidden') is not None else {}
456
+ pins = conf.get('pins', []) if conf.get('pins') is not None else []
457
+ if not isinstance(hidden, dict) or not isinstance(pins, (list, tuple)):
458
+ raise s_exc.BadArg(mesg='hidden must be an object and pins a list.')
459
+ commands = conf.get('commands', True)
460
+ if not isinstance(commands, bool):
461
+ raise s_exc.BadArg(mesg='commands must be true or false.')
462
+ out = {'hidden': {}, 'pins': [], 'commands': commands}
463
+ for key, rule in hidden.items():
464
+ if not isinstance(key, str) or not key or len(key) > 512 or not isinstance(rule, dict):
465
+ raise s_exc.BadArg(mesg='Each hidden entry maps an action (or pkg:<name>) to {all, roles}.')
466
+ roles = rule.get('roles') or []
467
+ if not isinstance(roles, (list, tuple)) or not all(isinstance(r, str) for r in roles):
468
+ raise s_exc.BadArg(mesg='roles must be a list of role idens.')
469
+ rule = {'all': rule.get('all') is True, 'roles': sorted(set(roles))}
470
+ if rule['all'] or rule['roles']:
471
+ out['hidden'][key] = rule
472
+ for iden in pins:
473
+ if not isinstance(iden, str) or not iden or len(iden) > 512:
474
+ raise s_exc.BadArg(mesg='pins must be a list of action idens.')
475
+ if iden not in out['pins']:
476
+ out['pins'].append(iden)
477
+ if len(out['hidden']) > 10000 or len(out['pins']) > 200:
478
+ raise s_exc.BadArg(mesg='Too many action settings.')
479
+ return out
480
+
481
+ def renameIdens(conf, aliases):
482
+ '''The settings with action idens renamed by aliases ({old iden: new}): hidden keys and pins.'''
483
+ hidden = {}
484
+ for key, rule in conf['hidden'].items():
485
+ hidden.setdefault(aliases.get(key, key), rule)
486
+ pins = []
487
+ for iden in conf['pins']:
488
+ iden = aliases.get(iden, iden)
489
+ if iden not in pins:
490
+ pins.append(iden)
491
+ return {**conf, 'hidden': hidden, 'pins': pins}
492
+
493
+ def isHidden(act, conf, roles):
494
+ '''True if the admin hid the action (or its package) from everyone, or from one of the user's roles.'''
495
+ for key in (act.get('iden'), f'pkg:{act.get("pkg")}'):
496
+ rule = conf['hidden'].get(key)
497
+ if rule and (rule['all'] or set(rule['roles']) & set(roles)):
498
+ return True
499
+ return False
500
+
501
+ def visibleActions(actions, conf, roles):
502
+ '''The actions a user with these role idens sees, as the admin set them.'''
503
+ offcmds = conf.get('commands') is False
504
+ return [a for a in actions if not isHidden(a, conf, roles) and not (offcmds and a.get('source') == 'command')]
neuralui/checkpkg.py ADDED
@@ -0,0 +1,60 @@
1
+ '''
2
+ Check a Storm package's neuralui: key, as a package author would before releasing it.
3
+
4
+ python -m neuralui.checkpkg path/to/pkgproto.yaml [more.yaml|pkgdef.json ...]
5
+
6
+ Each path is a package prototype (YAML, loaded as Synapse's genpkg loads it) or a built package definition
7
+ (JSON). For each, it checks the neuralui: key against the JSON Schema (neuralui/schema/neuralui.schema.json),
8
+ then builds the right-click actions as NeuralUI would and lists what it could not use and its warnings.
9
+ Exits 1 if anything is wrong, 0 if not.
10
+ '''
11
+ import sys
12
+ import json
13
+ import argparse
14
+
15
+ import synapse.tools.storm.pkg.gen as s_genpkg
16
+
17
+ import neuralui.actions as w_actions
18
+
19
+ def loadPkg(path):
20
+ if path.endswith('.json'):
21
+ with open(path) as fd:
22
+ return json.load(fd)
23
+ return json.loads(json.dumps(s_genpkg.tryLoadPkgProto(path, readonly=True)))
24
+
25
+ def check(pkgdef):
26
+ '''The problems with a package definition's neuralui: key, as lines of text (none when it is fine).'''
27
+ out = []
28
+ info = pkgdef.get('neuralui')
29
+ if info is not None:
30
+ mesg = w_actions.checkSchema(info)
31
+ if mesg:
32
+ out.append(f'schema: {mesg}')
33
+ reg = w_actions.buildRegistry([pkgdef])
34
+ for kind in ('errors', 'warnings'):
35
+ for item in reg[kind]:
36
+ out.append(f'{kind[:-1]}: {item["name"] or "(the package)"} ({item["source"]}): {item["mesg"]}')
37
+ return out
38
+
39
+ def main(argv):
40
+ pars = argparse.ArgumentParser(prog='python -m neuralui.checkpkg', description=__doc__.strip().split('\n')[0])
41
+ pars.add_argument('paths', nargs='+', help='Package prototypes (.yaml) or package definitions (.json).')
42
+ opts = pars.parse_args(argv)
43
+ bad = 0
44
+ for path in opts.paths:
45
+ try:
46
+ pkgdef = loadPkg(path)
47
+ except Exception as e:
48
+ print(f'{path}: could not load: {e}')
49
+ bad += 1
50
+ continue
51
+ lines = check(pkgdef)
52
+ acts = len(w_actions.buildRegistry([pkgdef])['actions'])
53
+ print(f'{path}: {pkgdef.get("name")}: {acts} action{"" if acts == 1 else "s"}' + ('' if lines else ', no problems'))
54
+ for line in lines:
55
+ print(f' {line}')
56
+ bad += bool(lines)
57
+ return 1 if bad else 0
58
+
59
+ if __name__ == '__main__': # pragma: no cover
60
+ sys.exit(main(sys.argv[1:]))