clue-api 1.0.0.dev7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. clue/.gitignore +21 -0
  2. clue/__init__.py +0 -0
  3. clue/api/__init__.py +211 -0
  4. clue/api/base.py +99 -0
  5. clue/api/v1/__init__.py +82 -0
  6. clue/api/v1/actions.py +92 -0
  7. clue/api/v1/auth.py +243 -0
  8. clue/api/v1/configs.py +83 -0
  9. clue/api/v1/fetchers.py +94 -0
  10. clue/api/v1/lookup.py +221 -0
  11. clue/api/v1/registration.py +109 -0
  12. clue/api/v1/static.py +94 -0
  13. clue/app.py +166 -0
  14. clue/cache/__init__.py +129 -0
  15. clue/common/__init__.py +0 -0
  16. clue/common/classification.py +1006 -0
  17. clue/common/classification.yml +130 -0
  18. clue/common/dict_utils.py +130 -0
  19. clue/common/exceptions.py +199 -0
  20. clue/common/forge.py +152 -0
  21. clue/common/json_utils.py +10 -0
  22. clue/common/list_utils.py +11 -0
  23. clue/common/logging/__init__.py +291 -0
  24. clue/common/logging/audit.py +157 -0
  25. clue/common/logging/format.py +42 -0
  26. clue/common/regex.py +31 -0
  27. clue/common/str_utils.py +213 -0
  28. clue/common/swagger.py +139 -0
  29. clue/common/uid.py +47 -0
  30. clue/config.py +60 -0
  31. clue/constants/__init__.py +0 -0
  32. clue/constants/supported_types.py +38 -0
  33. clue/cronjobs/__init__.py +30 -0
  34. clue/cronjobs/plugins.py +32 -0
  35. clue/error.py +129 -0
  36. clue/gunicorn_config.py +29 -0
  37. clue/healthz.py +74 -0
  38. clue/helper/discover.py +53 -0
  39. clue/helper/headers.py +30 -0
  40. clue/helper/oauth.py +128 -0
  41. clue/models/__init__.py +0 -0
  42. clue/models/actions.py +243 -0
  43. clue/models/config.py +456 -0
  44. clue/models/fetchers.py +136 -0
  45. clue/models/graph.py +162 -0
  46. clue/models/model_list.py +52 -0
  47. clue/models/network.py +430 -0
  48. clue/models/results/__init__.py +34 -0
  49. clue/models/results/base.py +10 -0
  50. clue/models/results/graph.py +26 -0
  51. clue/models/results/image.py +22 -0
  52. clue/models/results/status.py +55 -0
  53. clue/models/results/validation.py +57 -0
  54. clue/models/selector.py +67 -0
  55. clue/models/utils.py +52 -0
  56. clue/models/validators.py +19 -0
  57. clue/patched.py +8 -0
  58. clue/plugin/__init__.py +1008 -0
  59. clue/plugin/helpers/__init__.py +0 -0
  60. clue/plugin/helpers/central_server.py +27 -0
  61. clue/plugin/helpers/email_render.py +228 -0
  62. clue/plugin/helpers/token.py +34 -0
  63. clue/plugin/helpers/trino.py +103 -0
  64. clue/plugin/interactive.py +270 -0
  65. clue/plugin/models.py +19 -0
  66. clue/plugin/utils.py +78 -0
  67. clue/remote/__init__.py +0 -0
  68. clue/remote/datatypes/__init__.py +130 -0
  69. clue/remote/datatypes/cache.py +62 -0
  70. clue/remote/datatypes/events.py +118 -0
  71. clue/remote/datatypes/hash.py +193 -0
  72. clue/remote/datatypes/queues/__init__.py +0 -0
  73. clue/remote/datatypes/queues/comms.py +62 -0
  74. clue/remote/datatypes/set.py +96 -0
  75. clue/remote/datatypes/user_quota_tracker.py +54 -0
  76. clue/security/__init__.py +211 -0
  77. clue/security/obo.py +95 -0
  78. clue/security/utils.py +34 -0
  79. clue/services/action_service.py +186 -0
  80. clue/services/auth_service.py +348 -0
  81. clue/services/config_service.py +38 -0
  82. clue/services/fetcher_service.py +203 -0
  83. clue/services/jwt_service.py +233 -0
  84. clue/services/lookup_service.py +786 -0
  85. clue/services/type_service.py +165 -0
  86. clue/services/user_service.py +152 -0
  87. clue_api-1.0.0.dev7.dist-info/METADATA +111 -0
  88. clue_api-1.0.0.dev7.dist-info/RECORD +91 -0
  89. clue_api-1.0.0.dev7.dist-info/WHEEL +4 -0
  90. clue_api-1.0.0.dev7.dist-info/entry_points.txt +8 -0
  91. clue_api-1.0.0.dev7.dist-info/licenses/LICENSE +11 -0
@@ -0,0 +1,1006 @@
1
+ import itertools
2
+ import logging
3
+ from copy import copy
4
+ from typing import Any, KeysView
5
+
6
+ log = logging.getLogger("clue.classification")
7
+
8
+
9
+ class InvalidClassification(Exception):
10
+ pass
11
+
12
+
13
+ class InvalidDefinition(Exception):
14
+ pass
15
+
16
+
17
+ class Classification(object):
18
+ MIN_LVL = 1
19
+ MAX_LVL = 10000
20
+ NULL_LVL = 0
21
+ INVALID_LVL = 10001
22
+ NULL_CLASSIFICATION = "NULL"
23
+ INVALID_CLASSIFICATION = "INVALID"
24
+
25
+ def __init__(self, classification_definition: dict[str, Any]):
26
+ """Returns the classification class instantiated with the classification_definition
27
+
28
+ Args:
29
+ classification_definition: The classification definition dictionary,
30
+ see default classification.yml for an example.
31
+ """
32
+ banned_params_keys = ["name", "short_name", "lvl", "aliases", "auto_select", "css", "description"]
33
+ self.original_definition = classification_definition
34
+ self.levels_map: dict[str, int] = {}
35
+ self.levels_map_stl: dict[str | int, str] = {}
36
+ self.levels_map_lts: dict[str, str] = {}
37
+ self.levels_styles_map = {}
38
+ self.levels_aliases = {}
39
+ self.access_req_map_lts = {}
40
+ self.access_req_map_stl = {}
41
+ self.access_req_aliases = {}
42
+ self.groups_map_lts = {}
43
+ self.groups_map_stl = {}
44
+ self.groups_aliases = {}
45
+ self.groups_auto_select: list[str] = []
46
+ self.groups_auto_select_short: list[str] = []
47
+ self.subgroups_map_lts = {}
48
+ self.subgroups_map_stl = {}
49
+ self.subgroups_aliases = {}
50
+ self.subgroups_auto_select: list[str] = []
51
+ self.subgroups_auto_select_short: list[str] = []
52
+ self.params_map = {}
53
+ self.description = {}
54
+ self.invalid_mode = False
55
+ self._classification_cache = set()
56
+ self._classification_cache_short = set()
57
+
58
+ self.enforce = False
59
+ self.dynamic_groups = False
60
+ # dynamic group type is one of: email | group | all
61
+ # defaults to email for original behavior
62
+ self.dynamic_groups_type = "email"
63
+
64
+ # Add Invalid classification
65
+ self.levels_map["INV"] = self.INVALID_LVL
66
+ self.levels_map[str(self.INVALID_LVL)] = "INV"
67
+ self.levels_map_stl["INV"] = self.INVALID_CLASSIFICATION
68
+ self.levels_map_lts[self.INVALID_CLASSIFICATION] = "INV"
69
+
70
+ # Add null classification
71
+ self.levels_map[self.NULL_CLASSIFICATION] = self.NULL_LVL
72
+ self.levels_map[str(self.NULL_LVL)] = self.NULL_CLASSIFICATION
73
+ self.levels_map_stl[self.NULL_CLASSIFICATION] = self.NULL_CLASSIFICATION
74
+ self.levels_map_lts[self.NULL_CLASSIFICATION] = self.NULL_CLASSIFICATION
75
+
76
+ try:
77
+ self.enforce = classification_definition["enforce"]
78
+ self.dynamic_groups = classification_definition["dynamic_groups"]
79
+ self.dynamic_groups_type = classification_definition["dynamic_groups_type"]
80
+
81
+ if self.dynamic_groups_type not in ["email", "group", "all"]:
82
+ raise InvalidDefinition(
83
+ f'Invalid dynamic group type "{self.dynamic_groups_type}". ' "Valid types are: email | group | all"
84
+ )
85
+
86
+ for x in classification_definition["levels"]:
87
+ short_name = x["short_name"].upper()
88
+ name = x["name"].upper()
89
+
90
+ if short_name in ["INV", "NULL"] or name in [self.INVALID_CLASSIFICATION, self.NULL_CLASSIFICATION]:
91
+ raise InvalidDefinition(
92
+ "You cannot use reserved words NULL, INVALID or INV in your " "classification definition."
93
+ )
94
+
95
+ lvl = int(x["lvl"])
96
+ if lvl > self.MAX_LVL:
97
+ raise InvalidDefinition("Level over maximum classification level of %s." % self.MAX_LVL)
98
+ if lvl < self.MIN_LVL:
99
+ raise InvalidDefinition("Level under minimum classification level of %s." % self.MIN_LVL)
100
+
101
+ self.levels_map[short_name] = lvl
102
+ self.levels_map[str(lvl)] = short_name
103
+ self.levels_map_stl[short_name] = name
104
+ self.levels_map_lts[name] = short_name
105
+ for a in x.get("aliases", []):
106
+ self.levels_aliases[a.upper()] = short_name
107
+ self.params_map[short_name] = {k: v for k, v in x.items() if k not in banned_params_keys}
108
+ self.params_map[name] = self.params_map[short_name]
109
+ self.levels_styles_map[short_name] = x.get("css", {"color": "default"})
110
+ self.levels_styles_map[name] = self.levels_styles_map[short_name]
111
+ self.description[short_name] = x.get("description", "N/A")
112
+ self.description[name] = self.description[short_name]
113
+
114
+ for x in classification_definition["required"]:
115
+ short_name = x["short_name"].upper()
116
+ name = x["name"].upper()
117
+ self.access_req_map_lts[name] = short_name
118
+ self.access_req_map_stl[short_name] = name
119
+ for a in x.get("aliases", []):
120
+ self.access_req_aliases[a.upper()] = self.access_req_aliases.get(a.upper(), []) + [short_name]
121
+ self.params_map[short_name] = {k: v for k, v in x.items() if k not in banned_params_keys}
122
+ self.params_map[name] = self.params_map[short_name]
123
+ self.description[short_name] = x.get("description", "N/A")
124
+ self.description[name] = self.description[short_name]
125
+
126
+ for x in classification_definition["groups"]:
127
+ short_name = x["short_name"].upper()
128
+ name = x["name"].upper()
129
+ self.groups_map_lts[name] = short_name
130
+ self.groups_map_stl[short_name] = name
131
+ for a in x.get("aliases", []):
132
+ self.groups_aliases[a.upper()] = list(set(self.groups_aliases.get(a.upper(), []) + [short_name]))
133
+ solitary_display_name = x.get("solitary_display_name", None)
134
+ if solitary_display_name:
135
+ self.groups_aliases[solitary_display_name.upper()] = list(
136
+ set(self.groups_aliases.get(solitary_display_name.upper(), []) + [short_name])
137
+ )
138
+ if x.get("auto_select", False):
139
+ self.groups_auto_select.append(name)
140
+ self.groups_auto_select_short.append(short_name)
141
+ self.params_map[short_name] = {k: v for k, v in x.items() if k not in banned_params_keys}
142
+ self.params_map[name] = self.params_map[short_name]
143
+ self.description[short_name] = x.get("description", "N/A")
144
+ self.description[name] = self.description[short_name]
145
+
146
+ for x in classification_definition["subgroups"]:
147
+ short_name = x["short_name"].upper()
148
+ name = x["name"].upper()
149
+ self.subgroups_map_lts[name] = short_name
150
+ self.subgroups_map_stl[short_name] = name
151
+ for a in x.get("aliases", []):
152
+ self.subgroups_aliases[a.upper()] = list(
153
+ set(self.subgroups_aliases.get(a.upper(), []) + [short_name])
154
+ )
155
+ solitary_display_name = x.get("solitary_display_name", None)
156
+ if solitary_display_name:
157
+ self.subgroups_aliases[solitary_display_name.upper()] = list(
158
+ set(self.subgroups_aliases.get(solitary_display_name.upper(), []) + [short_name])
159
+ )
160
+ if x.get("auto_select", False):
161
+ self.subgroups_auto_select.append(name)
162
+ self.subgroups_auto_select_short.append(short_name)
163
+ self.params_map[short_name] = {k: v for k, v in x.items() if k not in banned_params_keys}
164
+ self.params_map[name] = self.params_map[short_name]
165
+ self.description[short_name] = x.get("description", "N/A")
166
+ self.description[name] = self.description[short_name]
167
+
168
+ if not self.is_valid(classification_definition["unrestricted"]):
169
+ raise InvalidDefinition("Classification definition's unrestricted classification is invalid.")
170
+
171
+ if not self.is_valid(classification_definition["restricted"]):
172
+ raise InvalidDefinition("Classification definition's restricted classification is invalid.")
173
+
174
+ self.UNRESTRICTED = classification_definition["unrestricted"]
175
+ self.RESTRICTED = classification_definition["restricted"]
176
+
177
+ self.UNRESTRICTED = self.normalize_classification(classification_definition["unrestricted"])
178
+ self.RESTRICTED = self.normalize_classification(classification_definition["restricted"])
179
+
180
+ except Exception as e:
181
+ self.UNRESTRICTED = self.NULL_CLASSIFICATION
182
+ self.RESTRICTED = self.INVALID_CLASSIFICATION
183
+
184
+ self.invalid_mode = True
185
+
186
+ log.warning(str(e))
187
+
188
+ ############################
189
+ # Private functions
190
+ ############################
191
+ @staticmethod
192
+ def _build_combinations(items: set, separator: str = "/", solitary_display: dict[str, Any] | None = None) -> set:
193
+ if solitary_display is None:
194
+ solitary_display = {}
195
+
196
+ out = {""}
197
+ for i in items:
198
+ others = [x for x in items if x != i]
199
+ for x in range(len(others) + 1):
200
+ for c in itertools.combinations(others, x):
201
+ value = separator.join(sorted([i] + list(c)))
202
+ out.add(solitary_display.get(value, value))
203
+
204
+ return out
205
+
206
+ @staticmethod
207
+ def _list_items_and_aliases(data: list[dict[str, Any]], long_format: bool = True) -> set[str]:
208
+ items = set()
209
+ for item in data:
210
+ if long_format:
211
+ items.add(item["name"])
212
+ else:
213
+ items.add(item["short_name"])
214
+
215
+ return items
216
+
217
+ def _get_c12n_level_index(self, c12n: str) -> tuple[int, str]:
218
+ # Parse classifications in uppercase mode only
219
+ c12n = c12n.upper()
220
+
221
+ lvl, _, remain = c12n.partition("//")
222
+ if lvl in self.levels_map:
223
+ return self.levels_map[lvl], remain
224
+ elif lvl in self.levels_map_lts:
225
+ return self.levels_map[self.levels_map_lts[lvl]], remain
226
+ elif lvl in self.levels_aliases:
227
+ return self.levels_map[self.levels_aliases[lvl]], remain
228
+ else:
229
+ raise InvalidClassification(
230
+ "Classification level '%s' was not found in " "your classification definition." % lvl
231
+ )
232
+
233
+ def _get_c12n_level_text(self, lvl_idx: int, long_format: bool = True) -> str:
234
+ text = self.levels_map.get(str(lvl_idx), None)
235
+ if not text:
236
+ raise InvalidClassification(
237
+ "Classification level number '%s' was not " "found in your classification definition." % lvl_idx
238
+ )
239
+ if long_format:
240
+ return self.levels_map_stl[text]
241
+
242
+ return text
243
+
244
+ def _get_c12n_required(self, c12n: str, long_format: bool = True) -> tuple[list, list[str]]:
245
+ # Parse classifications in uppercase mode only
246
+ c12n = c12n.upper()
247
+
248
+ return_set = set()
249
+ part_set = set(c12n.split("/"))
250
+ unused = []
251
+
252
+ for p in part_set:
253
+ if not p:
254
+ continue
255
+ if p in self.access_req_map_lts:
256
+ return_set.add(self.access_req_map_lts[p])
257
+ elif p in self.access_req_map_stl:
258
+ return_set.add(p)
259
+ elif p in self.access_req_aliases:
260
+ for a in self.access_req_aliases[p]:
261
+ return_set.add(a)
262
+ else:
263
+ unused.append(p)
264
+
265
+ if long_format:
266
+ return sorted([self.access_req_map_stl[r] for r in return_set]), unused
267
+ return sorted(list(return_set)), unused
268
+
269
+ def _get_c12n_groups(
270
+ self, c12n: list[str], long_format: bool = True, get_dynamic_groups: bool = True, auto_select: bool = False
271
+ ) -> tuple[list, list, list[str]]:
272
+ # Parse classifications in uppercase mode only
273
+
274
+ g1_set = set()
275
+ g2_set = set()
276
+ others = set()
277
+
278
+ groups = []
279
+ subgroups = []
280
+
281
+ for gp in c12n:
282
+ # If there is a rel marking we know we have groups
283
+ if gp.startswith("REL "):
284
+ gp = gp.replace("REL TO ", "")
285
+ gp = gp.replace("REL ", "")
286
+ temp_group = set([x.strip() for x in gp.split(",")])
287
+ for t in temp_group:
288
+ groups.extend(t.split("/"))
289
+ else:
290
+ # if there is not a rel marking we either have a subgroup or a solitary_display_name
291
+ # alias for a group, which we will filter out later
292
+ subgroups.append(gp)
293
+
294
+ for g in groups:
295
+ if g in self.groups_map_lts:
296
+ g1_set.add(self.groups_map_lts[g])
297
+ elif g in self.groups_map_stl:
298
+ g1_set.add(g)
299
+ elif g in self.groups_aliases:
300
+ for a in self.groups_aliases[g]:
301
+ g1_set.add(a)
302
+ else:
303
+ others.add(g)
304
+
305
+ for g in subgroups:
306
+ if g in self.subgroups_map_lts:
307
+ g2_set.add(self.subgroups_map_lts[g])
308
+ elif g in self.subgroups_map_stl:
309
+ g2_set.add(g)
310
+ elif g in self.subgroups_aliases:
311
+ for a in self.subgroups_aliases[g]:
312
+ g2_set.add(a)
313
+ # Here is where we catch any solitary_display_name aliases for groups within the subgroup sections
314
+ elif g in self.groups_aliases:
315
+ # Check that this alias is actually a solitary name, don't
316
+ # let other aliases leak outside the REL marking
317
+ groups = self.groups_aliases[g]
318
+ if len(groups) > 1:
319
+ raise InvalidClassification(f"Unclear use of alias: {g}")
320
+ g1_set.add(groups[0])
321
+ else:
322
+ raise InvalidClassification(f"Unknown Subgroup: {g}")
323
+
324
+ # If dynamic groups are active all remaining parts should be groups found under a
325
+ # REL TO marking that we can merge in with the other groups
326
+ if self.dynamic_groups and get_dynamic_groups:
327
+ g1_set.update(others)
328
+ others = set()
329
+
330
+ # Check if there are any required group assignments
331
+ for subgroup in g2_set:
332
+ required = self.params_map.get(subgroup, {}).get("require_group", None)
333
+ if required:
334
+ g1_set.add(required)
335
+
336
+ # Check if there are any forbidden group assignments
337
+ for subgroup in g2_set:
338
+ limited_to_group = self.params_map.get(subgroup, {}).get("limited_to_group", None)
339
+ if limited_to_group is not None:
340
+ if len(g1_set) > 1 or (len(g1_set) == 1 and g1_set != set([limited_to_group])):
341
+ raise InvalidClassification(
342
+ f"Subgroup {subgroup} is limited to group " f"{limited_to_group} (found: {', '.join(g1_set)})"
343
+ )
344
+
345
+ # Do auto select
346
+ if auto_select and g1_set:
347
+ g1_set.update(self.groups_auto_select_short)
348
+ if auto_select and g2_set:
349
+ g2_set.update(self.subgroups_auto_select_short)
350
+
351
+ # Swap to long format if required
352
+ if long_format:
353
+ return (
354
+ sorted([self.groups_map_stl.get(r, r) for r in g1_set]),
355
+ sorted([self.subgroups_map_stl[r] for r in g2_set]),
356
+ list(others),
357
+ )
358
+ return sorted(list(g1_set)), sorted(list(g2_set)), list(others)
359
+
360
+ @staticmethod
361
+ def _can_see_required(user_req: list, req: list) -> bool:
362
+ return set(req).issubset(user_req)
363
+
364
+ @staticmethod
365
+ def _can_see_groups(user_groups: list, req: list) -> bool:
366
+ if len(req) == 0:
367
+ return True
368
+
369
+ for g in user_groups:
370
+ if g in req:
371
+ return True
372
+
373
+ return False
374
+
375
+ # noinspection PyTypeChecker
376
+ def _get_normalized_classification_text(
377
+ self,
378
+ lvl_idx: int,
379
+ req: list,
380
+ groups: list,
381
+ subgroups: list,
382
+ long_format: bool = True,
383
+ skip_auto_select: bool = False,
384
+ ) -> str:
385
+ group_delim = "REL TO " if long_format else "REL "
386
+
387
+ # 1. Check for all required items if they need a specific classification lvl
388
+ required_lvl_idx = 0
389
+ for r in req:
390
+ required_lvl_idx = max(required_lvl_idx, self.params_map.get(r, {}).get("require_lvl", 0))
391
+ out = self._get_c12n_level_text(max(lvl_idx, required_lvl_idx), long_format=long_format)
392
+
393
+ # 2. Check for all required items if they should be shown inside the groups display part
394
+ req_grp = []
395
+ for r in req:
396
+ if self.params_map.get(r, {}).get("is_required_group"):
397
+ req_grp.append(r)
398
+ req = list(set(req).difference(set(req_grp)))
399
+
400
+ if req:
401
+ out += "//" + "/".join(sorted(req))
402
+ if req_grp:
403
+ out += "//" + "/".join(sorted(req_grp))
404
+
405
+ # 3. Add auto-selected subgroups
406
+ if long_format:
407
+ if len(subgroups) > 0 and len(self.subgroups_auto_select) > 0 and not skip_auto_select:
408
+ subgroups = sorted(list(set(subgroups).union(set(self.subgroups_auto_select))))
409
+ else:
410
+ if len(subgroups) > 0 and len(self.subgroups_auto_select_short) > 0 and not skip_auto_select:
411
+ subgroups = sorted(list(set(subgroups).union(set(self.subgroups_auto_select_short))))
412
+
413
+ # 4. For every subgroup, check if the subgroup requires or is limited to a specific group
414
+ temp_groups = []
415
+ for sg in subgroups:
416
+ required_group = self.params_map.get(sg, {}).get("require_group", None)
417
+ if required_group is not None:
418
+ temp_groups.append(required_group)
419
+
420
+ limited_to_group = self.params_map.get(sg, {}).get("limited_to_group", None)
421
+ if limited_to_group is not None:
422
+ if limited_to_group in temp_groups:
423
+ temp_groups = [limited_to_group]
424
+ else:
425
+ temp_groups = []
426
+
427
+ for g in temp_groups:
428
+ if long_format:
429
+ groups.append(self.groups_map_stl.get(g, g))
430
+ else:
431
+ groups.append(self.groups_map_lts.get(g, g))
432
+ groups = list(set(groups))
433
+
434
+ # 5. Add auto-selected groups
435
+ if long_format:
436
+ if len(groups) > 0 and len(self.groups_auto_select) > 0 and not skip_auto_select:
437
+ groups = sorted(list(set(groups).union(set(self.groups_auto_select))))
438
+ else:
439
+ if len(groups) > 0 and len(self.groups_auto_select_short) > 0 and not skip_auto_select:
440
+ groups = sorted(list(set(groups).union(set(self.groups_auto_select_short))))
441
+
442
+ if groups:
443
+ groups = sorted(groups)
444
+ out += {True: "/", False: "//"}[len(req_grp) > 0]
445
+ if len(groups) == 1:
446
+ # 6. If only one group, check if it has a solitary display name.
447
+ grp = groups[0]
448
+ display_name = self.params_map.get(grp, {}).get("solitary_display_name", grp)
449
+ if display_name != grp:
450
+ out += display_name
451
+ else:
452
+ out += group_delim + grp
453
+ else:
454
+ if not long_format:
455
+ # 7. In short format mode, check if there is an alias that can replace multiple groups
456
+ for alias, values in self.groups_aliases.items():
457
+ if len(values) > 1:
458
+ if sorted(values) == groups:
459
+ groups = [alias]
460
+ out += group_delim + ", ".join(sorted(groups))
461
+
462
+ if subgroups:
463
+ if len(groups) > 0 or len(req_grp) > 0:
464
+ out += "/"
465
+ else:
466
+ out += "//"
467
+ out += "/".join(sorted(subgroups))
468
+
469
+ return out.upper()
470
+
471
+ def _get_classification_parts(
472
+ self,
473
+ c12n: str,
474
+ long_format: bool = True,
475
+ get_dynamic_groups: bool = True,
476
+ auto_select: bool = False,
477
+ ignore_unused: bool = False,
478
+ ) -> tuple[int, list[str], list[str], list[str]]:
479
+ lvl_idx, unused = self._get_c12n_level_index(c12n)
480
+ req, unused_parts = self._get_c12n_required(unused, long_format=long_format)
481
+ groups, subgroups, unused_parts = self._get_c12n_groups(
482
+ unused_parts, long_format=long_format, get_dynamic_groups=get_dynamic_groups, auto_select=auto_select
483
+ )
484
+
485
+ if unused_parts and not ignore_unused:
486
+ raise InvalidClassification(f"Unparsable classification parts: {''.join(unused_parts)}")
487
+
488
+ return lvl_idx, req, groups, subgroups
489
+
490
+ @staticmethod
491
+ def _max_groups(groups_1: list, groups_2: list) -> list:
492
+ if len(groups_1) > 0 and len(groups_2) > 0:
493
+ groups = set(groups_1) & set(groups_2)
494
+ else:
495
+ groups = set(groups_1) | set(groups_2)
496
+
497
+ if len(groups_1) > 0 and len(groups_2) > 0 and len(groups) == 0:
498
+ # NOTE: Intersection generated nothing, we will raise an InvalidClassification exception
499
+ raise InvalidClassification(
500
+ "Could not find any intersection between the groups. %s & %s" % (groups_1, groups_2)
501
+ )
502
+
503
+ return list(groups)
504
+
505
+ # ++++++++++++++++++++++++
506
+ # Public functions
507
+ # ++++++++++++++++++++++++
508
+ # noinspection PyUnusedLocal
509
+ def list_all_classification_combinations(self, long_format: bool = True, normalized: bool = False) -> set:
510
+ """NOTE: Listing all classifcation permutations can take a really long time the more the classification
511
+ definition is complexe. Normalizing each entry makes it even worst. Use only this function if
512
+ absolutely necessary.
513
+ """
514
+ combinations: set[str] = set()
515
+
516
+ levels = self._list_items_and_aliases(self.original_definition["levels"], long_format=long_format)
517
+ reqs = self._list_items_and_aliases(self.original_definition["required"], long_format=long_format)
518
+ grps = self._list_items_and_aliases(self.original_definition["groups"], long_format=long_format)
519
+ sgrps = self._list_items_and_aliases(self.original_definition["subgroups"], long_format=long_format)
520
+
521
+ req_cbs = self._build_combinations(reqs)
522
+ if long_format:
523
+ grp_solitary_display = {
524
+ x["name"]: x["solitary_display_name"]
525
+ for x in self.original_definition["groups"]
526
+ if "solitary_display_name" in x
527
+ }
528
+ else:
529
+ grp_solitary_display = {
530
+ x["short_name"]: x["solitary_display_name"]
531
+ for x in self.original_definition["groups"]
532
+ if "solitary_display_name" in x
533
+ }
534
+ solitary_names = [
535
+ x["solitary_display_name"] for x in self.original_definition["groups"] if "solitary_display_name" in x
536
+ ]
537
+
538
+ grp_cbs = self._build_combinations(grps, separator=", ", solitary_display=grp_solitary_display)
539
+ sgrp_cbs = self._build_combinations(sgrps)
540
+
541
+ for p in itertools.product(levels, req_cbs):
542
+ cl = "//".join(p)
543
+ if cl.endswith("//"):
544
+ combinations.add(cl[:-2])
545
+ else:
546
+ combinations.add(cl)
547
+
548
+ temp_combinations = copy(combinations)
549
+ for p in itertools.product(temp_combinations, grp_cbs):
550
+ cl = "//REL TO ".join(p)
551
+ if cl.endswith("//REL TO "):
552
+ combinations.add(cl[:-9])
553
+ else:
554
+ combinations.add(cl)
555
+
556
+ for sol_name in solitary_names:
557
+ to_edit = []
558
+ to_find = "REL TO {sol_name}".format(sol_name=sol_name)
559
+ for c in combinations:
560
+ if to_find in c:
561
+ to_edit.append(c)
562
+
563
+ for e in to_edit:
564
+ combinations.add(e.replace(to_find, sol_name))
565
+ combinations.remove(e)
566
+
567
+ temp_combinations = copy(combinations)
568
+ for p in itertools.product(temp_combinations, sgrp_cbs):
569
+ if "//REL TO " in p[0]:
570
+ cl = "/".join(p)
571
+
572
+ if cl.endswith("/"):
573
+ combinations.add(cl[:-1])
574
+ else:
575
+ combinations.add(cl)
576
+ else:
577
+ cl = "//REL TO ".join(p)
578
+
579
+ if cl.endswith("//REL TO "):
580
+ combinations.add(cl[:-9])
581
+ else:
582
+ combinations.add(cl)
583
+
584
+ if normalized:
585
+ good = []
586
+ for x in combinations:
587
+ try:
588
+ good.append(self.normalize_classification(x, long_format=long_format))
589
+ except InvalidClassification:
590
+ pass
591
+ return set(good)
592
+ return combinations
593
+
594
+ # noinspection PyUnusedLocal
595
+ def default_user_classification(self, user: str | None = None, long_format: bool = True) -> str:
596
+ """You can overload this function to specify a way to get the default classification of a user.
597
+ By default, this function returns the UNRESTRICTED value of your classification definition.
598
+
599
+ Args:
600
+ user: Which user to get the classification for
601
+ long_format: Request a long classification format or not
602
+
603
+ Returns:
604
+ The classification in the specified format
605
+ """
606
+ return self.UNRESTRICTED
607
+
608
+ def get_parsed_classification_definition(self) -> dict[str, Any]:
609
+ """Returns all dictionary of all the variables inside the classification object that will be used
610
+ to enforce classification throughout the system.
611
+ """
612
+ from copy import deepcopy
613
+
614
+ out = deepcopy(self.__dict__)
615
+ out["levels_map"].pop("INV", None)
616
+ out["levels_map"].pop(str(self.INVALID_LVL), None)
617
+ out["levels_map_stl"].pop("INV", None)
618
+ out["levels_map_lts"].pop("INVALID", None)
619
+ out["levels_map"].pop("NULL", None)
620
+ out["levels_map"].pop(str(self.NULL_LVL), None)
621
+ out["levels_map_stl"].pop("NULL", None)
622
+ out["levels_map_lts"].pop("NULL", None)
623
+ out.pop("_classification_cache", None)
624
+ out.pop("_classification_cache_short", None)
625
+ return out
626
+
627
+ def get_access_control_parts(self, c12n: str, user_classification: bool = False) -> dict[str, Any]:
628
+ """Returns a dictionary containing the different access parameters Lucene needs to build it's queries
629
+
630
+ Args:
631
+ c12n: The classification to get the parts from
632
+ user_classification: Is a user classification
633
+ """
634
+ if not self.enforce or self.invalid_mode:
635
+ c12n = self.UNRESTRICTED
636
+
637
+ try:
638
+ parts = self._get_classification_parts(c12n, long_format=False, auto_select=not user_classification)
639
+ access_lvl, access_req, access_grp1, access_grp2 = parts
640
+
641
+ return {
642
+ "__access_lvl__": access_lvl,
643
+ "__access_req__": access_req,
644
+ "__access_grp1__": access_grp1 or ["__EMPTY__"],
645
+ "__access_grp2__": access_grp2 or ["__EMPTY__"],
646
+ }
647
+ except InvalidClassification:
648
+ if not self.enforce or self.invalid_mode:
649
+ return {
650
+ "__access_lvl__": self.NULL_LVL,
651
+ "__access_req__": [],
652
+ "__access_grp1__": ["__EMPTY__"],
653
+ "__access_grp2__": ["__EMPTY__"],
654
+ }
655
+ else:
656
+ raise
657
+
658
+ def get_access_control_req(self) -> KeysView | list:
659
+ """Returns a list of the different possible REQUIRED parts"""
660
+ if not self.enforce or self.invalid_mode:
661
+ return []
662
+
663
+ return self.access_req_map_stl.keys()
664
+
665
+ def get_access_control_groups(self) -> KeysView | list:
666
+ """Returns a list of the different possible GROUPS"""
667
+ if not self.enforce or self.invalid_mode:
668
+ return []
669
+
670
+ return self.groups_map_stl.keys()
671
+
672
+ def get_access_control_subgroups(self) -> KeysView | list:
673
+ """Returns a list of the different possible SUBGROUPS"""
674
+ if not self.enforce or self.invalid_mode:
675
+ return []
676
+
677
+ return self.subgroups_map_stl.keys()
678
+
679
+ def intersect_user_classification(self, user_c12n_1: str, user_c12n_2: str, long_format: bool = True) -> str:
680
+ """This function intersects two user classification to return the maximum classification
681
+ that both user could see.
682
+
683
+ Args:
684
+ user_c12n_1: First user classification
685
+ user_c12n_2: Second user classification
686
+ long_format: True/False in long format
687
+
688
+ Returns:
689
+ Intersected classification in the desired format
690
+ """
691
+ if not self.enforce or self.invalid_mode:
692
+ return self.UNRESTRICTED
693
+
694
+ if user_c12n_1 is None:
695
+ return user_c12n_2
696
+ if user_c12n_2 is None:
697
+ return user_c12n_1
698
+
699
+ lvl_idx_1, req_1, groups_1, subgroups_1 = self._get_classification_parts(user_c12n_1, long_format=long_format)
700
+ lvl_idx_2, req_2, groups_2, subgroups_2 = self._get_classification_parts(user_c12n_2, long_format=long_format)
701
+
702
+ req = list(set(req_1) & set(req_2))
703
+ groups = list(set(groups_1) & set(groups_2))
704
+ subgroups = list(set(subgroups_1) & set(subgroups_2))
705
+
706
+ return self._get_normalized_classification_text(
707
+ min(lvl_idx_1, lvl_idx_2), req, groups, subgroups, long_format=long_format, skip_auto_select=True
708
+ )
709
+
710
+ def is_accessible(self, user_c12n: str, c12n: str | None, ignore_invalid: bool = False) -> bool:
711
+ """Given a user classification, check if a user is allow to see a certain classification
712
+
713
+ Args:
714
+ user_c12n: Maximum classification for the user
715
+ c12n: Classification the user which to see
716
+
717
+ Returns:
718
+ True is the user can see the classification
719
+ """
720
+ if self.invalid_mode:
721
+ return False
722
+
723
+ if not self.enforce or c12n is None:
724
+ return True
725
+
726
+ try:
727
+ user_lvl, user_req, user_groups, user_subgroups = self._get_classification_parts(user_c12n)
728
+ lvl, req, groups, subgroups = self._get_classification_parts(c12n)
729
+
730
+ if int(user_lvl) >= int(lvl):
731
+ if not self._can_see_required(user_req, req):
732
+ return False
733
+ if not self._can_see_groups(user_groups, groups):
734
+ return False
735
+ if not self._can_see_groups(user_subgroups, subgroups):
736
+ return False
737
+ return True
738
+ return False
739
+ except InvalidClassification:
740
+ if ignore_invalid:
741
+ return False
742
+ else:
743
+ raise
744
+
745
+ def is_valid(self, c12n: str, skip_auto_select: bool = False) -> bool:
746
+ """Performs a series of checks againts a classification to make sure it is valid in it's current form
747
+
748
+ Args:
749
+ c12n: The classification we want to validate
750
+ skip_auto_select: skip the auto selection phase
751
+
752
+ Returns:
753
+ True if the classification is valid
754
+ """
755
+ if not self.enforce:
756
+ return True
757
+
758
+ try:
759
+ # Classification normalization test
760
+ n_c12n = self.normalize_classification(c12n, skip_auto_select=skip_auto_select)
761
+ n_lvl_idx, n_req, n_groups, n_subgroups = self._get_classification_parts(n_c12n)
762
+ lvl_idx, req, groups, subgroups = self._get_classification_parts(c12n)
763
+ except InvalidClassification as e:
764
+ log.warning("%s is invalid due to: %s", c12n, str(e))
765
+ return False
766
+
767
+ if lvl_idx != n_lvl_idx:
768
+ log.warning("%s is invalid due to lvl_idx != n_lvl_idx", c12n)
769
+ return False
770
+
771
+ if sorted(req) != sorted(n_req):
772
+ log.warning("%s is invalid due to sorted(req) != sorted(n_req)", c12n)
773
+ return False
774
+
775
+ if sorted(groups) != sorted(n_groups):
776
+ log.warning("%s is invalid due to sorted(groups) != sorted(n_groups)", c12n)
777
+ return False
778
+
779
+ if sorted(subgroups) != sorted(n_subgroups):
780
+ log.warning("%s is invalid due to sorted(subgroups) != sorted(n_subgroups)", c12n)
781
+ return False
782
+
783
+ c12n = c12n.upper().replace("REL TO ", "").replace("REL ", "")
784
+ parts = c12n.split("//")
785
+
786
+ # There is a maximum of 3 parts
787
+ if len(parts) > 3:
788
+ log.warning("%s is invalid due to len(parts) > 3", c12n)
789
+ return False
790
+
791
+ cur_part = parts.pop(0)
792
+ # First parts as to be a classification level part
793
+ if (
794
+ cur_part not in self.levels_aliases.keys()
795
+ and cur_part not in self.levels_map_lts.keys()
796
+ and cur_part not in self.levels_map_stl.keys()
797
+ ):
798
+ log.warning(
799
+ "%s is invalid due to %s not in any of %s, %s, %s",
800
+ c12n,
801
+ cur_part,
802
+ f"[{','.join(str(entry) for entry in self.levels_aliases.keys())}]",
803
+ f"[{','.join(str(entry) for entry in self.levels_map_lts.keys())}]",
804
+ f"[{','.join(str(entry) for entry in self.levels_map_stl.keys())}]",
805
+ )
806
+ return False
807
+
808
+ check_groups = False
809
+ while len(parts) > 0:
810
+ # Can't be two groups sections.
811
+ if check_groups:
812
+ return False
813
+
814
+ cur_part = parts.pop(0)
815
+ items = cur_part.split("/")
816
+ comma_idx = None
817
+ for idx, i in enumerate(items):
818
+ if "," in i:
819
+ comma_idx = idx
820
+
821
+ if comma_idx is not None:
822
+ items += [x.strip() for x in items.pop(comma_idx).split(",")]
823
+
824
+ for i in items:
825
+ if not check_groups:
826
+ # If current item not found in access req, we might already be dealing with groups
827
+ if (
828
+ i not in self.access_req_aliases.keys()
829
+ and i not in self.access_req_map_stl.keys()
830
+ and i not in self.access_req_map_lts.keys()
831
+ ):
832
+ check_groups = True
833
+
834
+ if check_groups and not self.dynamic_groups:
835
+ # If not groups. That stuff does not exists...
836
+ if (
837
+ i not in self.groups_aliases.keys()
838
+ and i not in self.groups_map_stl.keys()
839
+ and i not in self.groups_map_lts.keys()
840
+ and i not in self.subgroups_aliases.keys()
841
+ and i not in self.subgroups_map_stl.keys()
842
+ and i not in self.subgroups_map_lts.keys()
843
+ ):
844
+ return False
845
+
846
+ return True
847
+
848
+ def max_classification(self, c12n_1: str, c12n_2: str, long_format: bool = True) -> str:
849
+ """Mixes to classification and returns to most restrictive form for them
850
+
851
+ Args:
852
+ c12n_1: First classification
853
+ c12n_2: Second classification
854
+ long_format: True/False in long format
855
+
856
+ Returns:
857
+ The most restrictive classification that we could create out of the two
858
+ """
859
+ if not self.enforce or self.invalid_mode:
860
+ return self.UNRESTRICTED
861
+
862
+ if c12n_1 is None:
863
+ return c12n_2
864
+ if c12n_2 is None:
865
+ return c12n_1
866
+
867
+ lvl_idx_1, req_1, groups_1, subgroups_1 = self._get_classification_parts(
868
+ c12n_1, long_format=long_format, auto_select=True
869
+ )
870
+ lvl_idx_2, req_2, groups_2, subgroups_2 = self._get_classification_parts(
871
+ c12n_2, long_format=long_format, auto_select=True
872
+ )
873
+
874
+ req = list(set(req_1) | set(req_2))
875
+ groups = self._max_groups(groups_1, groups_2)
876
+ subgroups = self._max_groups(subgroups_1, subgroups_2)
877
+
878
+ return self._get_normalized_classification_text(
879
+ max(lvl_idx_1, lvl_idx_2), req, groups, subgroups, long_format=long_format
880
+ )
881
+
882
+ def min_classification(self, c12n_1: str, c12n_2: str, long_format: bool = True) -> str:
883
+ """Mixes to classification and returns to least restrictive form for them
884
+
885
+ Args:
886
+ c12n_1: First classification
887
+ c12n_2: Second classification
888
+ long_format: True/False in long format
889
+
890
+ Returns:
891
+ The least restrictive classification that we could create out of the two
892
+ """
893
+ if not self.enforce or self.invalid_mode:
894
+ return self.UNRESTRICTED
895
+
896
+ if c12n_1 is None:
897
+ return c12n_2
898
+ if c12n_2 is None:
899
+ return c12n_1
900
+
901
+ lvl_idx_1, req_1, groups_1, subgroups_1 = self._get_classification_parts(
902
+ c12n_1, long_format=long_format, auto_select=True
903
+ )
904
+ lvl_idx_2, req_2, groups_2, subgroups_2 = self._get_classification_parts(
905
+ c12n_2, long_format=long_format, auto_select=True
906
+ )
907
+
908
+ req = list(set(req_1) & set(req_2))
909
+ if len(groups_1) > 0 and len(groups_2) > 0:
910
+ groups = list(set(groups_1) | set(groups_2))
911
+ else:
912
+ groups = []
913
+
914
+ if len(subgroups_1) > 0 and len(subgroups_2) > 0:
915
+ subgroups = list(set(subgroups_1) | set(subgroups_2))
916
+ else:
917
+ subgroups = []
918
+
919
+ return self._get_normalized_classification_text(
920
+ min(lvl_idx_1, lvl_idx_2), req, groups, subgroups, long_format=long_format
921
+ )
922
+
923
+ def normalize_classification(
924
+ self,
925
+ c12n: str,
926
+ long_format: bool = True,
927
+ skip_auto_select: bool = False,
928
+ get_dynamic_groups: bool = True,
929
+ ignore_unused: bool = False,
930
+ ) -> str:
931
+ """Normalize a given classification by applying the rules defined in the classification definition.
932
+ This function will remove any invalid parts and add missing parts to the classification.
933
+ It will also ensure that the display of the classification is always done the same way
934
+
935
+ Args:
936
+ c12n: Classification to normalize
937
+ long_format: True/False in long format
938
+ skip_auto_select: True/False skip group auto adding, use True when dealing with user's classifications
939
+
940
+ Returns:
941
+ A normalized version of the original classification
942
+ """
943
+ if not self.enforce or self.invalid_mode:
944
+ return self.UNRESTRICTED
945
+
946
+ c12n = c12n.upper()
947
+
948
+ # Has the classification has already been normalized before?
949
+ if long_format and c12n in self._classification_cache and get_dynamic_groups:
950
+ return c12n
951
+ if not long_format and c12n in self._classification_cache_short and get_dynamic_groups:
952
+ return c12n
953
+
954
+ lvl_idx, req, groups, subgroups = self._get_classification_parts(
955
+ c12n, long_format=long_format, get_dynamic_groups=get_dynamic_groups, ignore_unused=ignore_unused
956
+ )
957
+ new_c12n = self._get_normalized_classification_text(
958
+ lvl_idx, req, groups, subgroups, long_format=long_format, skip_auto_select=skip_auto_select
959
+ )
960
+
961
+ if long_format:
962
+ self._classification_cache.add(new_c12n)
963
+ else:
964
+ self._classification_cache_short.add(new_c12n)
965
+
966
+ return new_c12n
967
+
968
+ def build_user_classification(self, c12n_1: str, c12n_2: str, long_format: bool = True) -> str:
969
+ """Mixes to classification and return the classification marking that would give access to the most data
970
+
971
+ Args:
972
+ c12n_1: First classification
973
+ c12n_2: Second classification
974
+ long_format: True/False in long format
975
+
976
+ Returns:
977
+ The classification that would give access to the most data
978
+ """
979
+ if not self.enforce or self.invalid_mode:
980
+ return self.UNRESTRICTED
981
+
982
+ if c12n_1 is None:
983
+ return c12n_2
984
+ if c12n_2 is None:
985
+ return c12n_1
986
+
987
+ lvl_idx_1, req_1, groups_1, subgroups_1 = self._get_classification_parts(c12n_1, long_format=long_format)
988
+ lvl_idx_2, req_2, groups_2, subgroups_2 = self._get_classification_parts(c12n_2, long_format=long_format)
989
+
990
+ req = list(set(req_1) | set(req_2))
991
+ groups = list(set(groups_1) | set(groups_2))
992
+ subgroups = list(set(subgroups_1) | set(subgroups_2))
993
+
994
+ return self._get_normalized_classification_text(
995
+ max(lvl_idx_1, lvl_idx_2), req, groups, subgroups, long_format=long_format, skip_auto_select=True
996
+ )
997
+
998
+
999
+ if __name__ == "__main__":
1000
+ from pprint import pprint
1001
+
1002
+ from clue.common import forge
1003
+
1004
+ classification = forge.get_classification()
1005
+ pprint(classification._classification_cache)
1006
+ pprint(classification._classification_cache_short)