streamlit-octostar-utils 0.2.9__tar.gz → 0.2.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/PKG-INFO +4 -2
  2. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/pyproject.toml +1 -1
  3. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/fastapi.py +1 -1
  4. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/nifi.py +86 -157
  5. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/nlp/ner.py +1 -4
  6. streamlit_octostar_utils-0.2.11/streamlit_octostar_utils/ontology/inheritance.py +13 -0
  7. streamlit_octostar_utils-0.2.9/streamlit_octostar_utils/ontology/inheritance.py +0 -2
  8. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/LICENSE +0 -0
  9. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/README.md +0 -0
  10. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/__init__.py +0 -0
  11. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/__init__.py +0 -0
  12. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/celery.py +0 -0
  13. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/__init__.py +0 -0
  14. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/combine_fields.py +0 -0
  15. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/entities_parser.py +0 -0
  16. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/generics.py +0 -0
  17. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/info.py +0 -0
  18. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/linkchart_functions.py +0 -0
  19. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/matches.py +0 -0
  20. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/parameters.py +0 -0
  21. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/rules.py +0 -0
  22. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/api_crafter/parser/signals.py +0 -0
  23. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/core/__init__.py +0 -0
  24. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/core/dict.py +0 -0
  25. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/core/filetypes.py +0 -0
  26. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/core/threading/__init__.py +0 -0
  27. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/core/threading/key_queue.py +0 -0
  28. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/core/timestamp.py +0 -0
  29. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/nlp/__init__.py +0 -0
  30. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/nlp/language.py +0 -0
  31. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/octostar/__init__.py +0 -0
  32. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/octostar/client.py +0 -0
  33. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/octostar/context.py +0 -0
  34. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/octostar/permissions.py +0 -0
  35. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/ontology/__init__.py +0 -0
  36. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/ontology/expand_entities.py +0 -0
  37. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/ontology/validation.py +0 -0
  38. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/style/__init__.py +0 -0
  39. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/style/common.py +0 -0
  40. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/threading/__init__.py +0 -0
  41. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/threading/async_task_manager.py +0 -0
  42. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/threading/session_callback_manager.py +0 -0
  43. {streamlit_octostar_utils-0.2.9 → streamlit_octostar_utils-0.2.11}/streamlit_octostar_utils/threading/session_state_hot_swapper.py +0 -0
@@ -1,8 +1,9 @@
1
- Metadata-Version: 2.3
1
+ Metadata-Version: 2.4
2
2
  Name: streamlit-octostar-utils
3
- Version: 0.2.9
3
+ Version: 0.2.11
4
4
  Summary:
5
5
  License: MIT
6
+ License-File: LICENSE
6
7
  Author: Octostar
7
8
  Requires-Python: >=3.9, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*, !=3.8.*
8
9
  Classifier: License :: OSI Approved :: MIT License
@@ -12,6 +13,7 @@ Classifier: Programming Language :: Python :: 3.10
12
13
  Classifier: Programming Language :: Python :: 3.11
13
14
  Classifier: Programming Language :: Python :: 3.12
14
15
  Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
15
17
  Provides-Extra: nlp
16
18
  Requires-Dist: PyJWT (>=2.5.0,<3.0.0)
17
19
  Requires-Dist: celery (>=5.3.0,<6.0.0)
@@ -5,7 +5,7 @@ include = '\.pyi?$'
5
5
 
6
6
  [tool.poetry]
7
7
  name = "streamlit-octostar-utils"
8
- version = "0.2.9"
8
+ version = "0.2.11"
9
9
  description = ""
10
10
  license = "MIT"
11
11
  authors = ["Octostar"]
@@ -330,7 +330,7 @@ class DefaultErrorRoute:
330
330
  if len(message) > MAX_ERROR_MESSAGE_BYTES:
331
331
  message = message[-MAX_ERROR_MESSAGE_BYTES:]
332
332
  try:
333
- tcbk = traceback.format_exception(exc)
333
+ tcbk = "\n".join(traceback.format_exception(exc))
334
334
  if len(tcbk) > MAX_ERROR_TRACEBACK_BYTES:
335
335
  tcbk = tcbk[-MAX_ERROR_TRACEBACK_BYTES:]
336
336
  except:
@@ -23,7 +23,7 @@ from octostar.client import make_client
23
23
  from ..core.dict import recursive_update_dict, travel_dict, jsondict_hash
24
24
  from ..core.timestamp import now, string_to_datetime
25
25
  from .fastapi import DefaultErrorRoute, Route
26
- from ..ontology.inheritance import is_child_concept as is_child_concept_fn
26
+ from ..ontology.inheritance import is_child_concept as is_child_concept_fn, get_label_keys
27
27
  from ..ontology.expand_entities import expand_entities
28
28
 
29
29
  RELATIONSHIP_ENTITY_NAME = "os_relationship"
@@ -69,6 +69,7 @@ class NifiEntityModel(BaseModel):
69
69
  class OntologyInfoModel(BaseModel):
70
70
  parents: List[str]
71
71
  relationships: List[str]
72
+ label_keys: List[str]
72
73
 
73
74
  class ContentsPointerModel(BaseModel):
74
75
  location: NifiContentsPointerLocationModel
@@ -162,16 +163,12 @@ class NifiEntityProxy(object):
162
163
  if child_entity.uid == uid_to_search:
163
164
  found_entity = child_entity
164
165
  else:
165
- found_entity = _recursive_search_expanded_proxy(
166
- child_entity._proxy, uid_to_search
167
- )
166
+ found_entity = _recursive_search_expanded_proxy(child_entity._proxy, uid_to_search)
168
167
  if found_entity:
169
168
  return found_entity
170
169
 
171
170
  if not self._proxy:
172
- main_entities = itertools.chain(
173
- *[b.entities for b in self.context.in_batches]
174
- )
171
+ main_entities = itertools.chain(*[b.entities for b in self.context.in_batches])
175
172
  main_entities = {e.record["entity_id"]: e for e in main_entities}
176
173
  if main_entities.get(self.uid):
177
174
  self._proxy = main_entities.get(self.uid)
@@ -182,9 +179,7 @@ class NifiEntityProxy(object):
182
179
  self._proxy = found_entity._proxy
183
180
  return self._proxy
184
181
  ## TODO: Try to get the entity from the database with query_ontology()
185
- raise AttributeError(
186
- f"Cannot find children with UUID {self.uid}! It may exist in the database?"
187
- )
182
+ raise AttributeError(f"Cannot find children with UUID {self.uid}! It may exist in the database?")
188
183
 
189
184
  def __getattr__(self, name):
190
185
  if name in self.__dict__:
@@ -218,28 +213,24 @@ class NifiFragmenter(object):
218
213
  raise ValueError("Must have at least 2 entities for fragmentation")
219
214
  identifier = str(uuid.uuid4())
220
215
  for i, entity in enumerate(fragments):
221
- travel_dict(
222
- entity.request["nifi_attributes"], fragmenter_keylist.split("."), "w"
223
- )({"identifier": identifier, "count": count, "index": i})
216
+ travel_dict(entity.request["nifi_attributes"], fragmenter_keylist.split("."), "w")(
217
+ {"identifier": identifier, "count": count, "index": i}
218
+ )
224
219
  if "fragment" not in entity.request["config"]:
225
220
  entity.request["config"]["fragment"] = {}
226
221
  if "fragments_stack" not in entity.request["config"]["fragment"]:
227
222
  entity.request["config"]["fragment"]["fragments_stack"] = []
228
- entity.request["config"]["fragment"]["fragments_stack"].insert(
229
- 0, fragmenter_keylist
223
+ entity.request["config"]["fragment"]["fragments_stack"].insert(0, fragmenter_keylist)
224
+ entity.request["nifi_attributes"]["fragments_stack"] = entity.request["config"]["fragment"][
225
+ "fragments_stack"
226
+ ]
227
+ travel_dict(entity.request["config"]["fragment"], fragmenter_keylist.split("."), "w")(
228
+ {"identifier": identifier, "count": count, "index": i}
230
229
  )
231
- entity.request["nifi_attributes"]["fragments_stack"] = entity.request[
232
- "config"
233
- ]["fragment"]["fragments_stack"]
234
- travel_dict(
235
- entity.request["config"]["fragment"], fragmenter_keylist.split("."), "w"
236
- )({"identifier": identifier, "count": count, "index": i})
237
230
 
238
231
  def push_defragment_strategy(fragment, defragmenter_config):
239
232
  pointer = fragment.request["config"]
240
- last_fragmenter_keylist = fragment.request["config"]["fragment"][
241
- "fragments_stack"
242
- ][0]
233
+ last_fragmenter_keylist = fragment.request["config"]["fragment"]["fragments_stack"][0]
243
234
  for k in ("fragment." + last_fragmenter_keylist).split("."):
244
235
  if not pointer.get(k):
245
236
  pointer[k] = {}
@@ -258,7 +249,7 @@ class NifiEntityBatch(object):
258
249
 
259
250
  class NifiContextManager(object):
260
251
  HEADLESS_PROCESSOR_NAME = "headless"
261
-
252
+
262
253
  class SyncFlag(Enum):
263
254
  UPSERT_ENTITY_ALL = 0 # bool
264
255
  UPSERT_ENTITY_SPECIFIC_FIELDS = 1 # 'fields': list of record fields
@@ -279,9 +270,7 @@ class NifiContextManager(object):
279
270
  @property
280
271
  def ontology(self):
281
272
  if not self._ontology:
282
- self._ontology = fetch_ontology_data.sync(
283
- ontology_name=self.ontology_name, client=self.client
284
- )
273
+ self._ontology = fetch_ontology_data.sync(ontology_name=self.ontology_name, client=self.client)
285
274
  return self._ontology
286
275
 
287
276
  def _config_get(entity, keylist):
@@ -350,9 +339,7 @@ class NifiContextManager(object):
350
339
  return self
351
340
 
352
341
  def get_workspaces_permissions(self, workspace_ids):
353
- permissions_to_fetch = list(
354
- set(workspace_ids).difference(set(list(self.permissions.keys())))
355
- )
342
+ permissions_to_fetch = list(set(workspace_ids).difference(set(list(self.permissions.keys()))))
356
343
  if permissions_to_fetch:
357
344
  permissions = get_permissions.sync(permissions_to_fetch, client=self.client)
358
345
  self.permissions.update(permissions)
@@ -382,18 +369,13 @@ class NifiContextManager(object):
382
369
  entities.append(entity)
383
370
  for child_entity in entity.children_entities:
384
371
  if not child_entity.drop_on_output:
385
- if (
386
- child_entity.output_as_independent
387
- or child_entity.output_as_child
388
- ):
372
+ if child_entity.output_as_independent or child_entity.output_as_child:
389
373
  if processor_name != NifiContextManager.HEADLESS_PROCESSOR_NAME:
390
374
  child_entity.request["last_processor_name"] = processor_name
391
375
  if child_entity.output_as_independent:
392
376
  if not child_entity._proxy:
393
377
  child_entity.fetch_proxy()
394
- entities.extend(
395
- _process_entity(child_entity._proxy, processor_name)
396
- )
378
+ entities.extend(_process_entity(child_entity._proxy, processor_name))
397
379
  return entities
398
380
 
399
381
  entities = itertools.chain(*[b.entities for b in entity_batches])
@@ -404,9 +386,7 @@ class NifiContextManager(object):
404
386
  all_entities,
405
387
  key=lambda x: string_to_datetime(x.record.get("os_last_updated_at")),
406
388
  )
407
- self.out_entities = list(
408
- {e.record["entity_id"]: e for e in all_entities}.values()
409
- )
389
+ self.out_entities = list({e.record["entity_id"]: e for e in all_entities}.values())
410
390
  self.sync_entities()
411
391
  return [entity for entity in self.jsonify(self.out_entities)["content"]]
412
392
 
@@ -414,23 +394,19 @@ class NifiContextManager(object):
414
394
  error_response = DefaultErrorRoute.format_error(exc)
415
395
  entity.request["exception"]["code"] = error_response.status_code
416
396
  entity.request["exception"]["body"] = json.loads(error_response.body)["message"]
417
- travel_dict(
418
- entity.request["nifi_attributes"], ["invokehttp", "response", "body"], "w"
419
- )(entity.request["exception"]["body"])
420
- travel_dict(
421
- entity.request["nifi_attributes"], ["invokehttp", "response", "code"], "w"
422
- )(entity.request["exception"]["code"])
397
+ travel_dict(entity.request["nifi_attributes"], ["invokehttp", "response", "body"], "w")(
398
+ entity.request["exception"]["body"]
399
+ )
400
+ travel_dict(entity.request["nifi_attributes"], ["invokehttp", "response", "code"], "w")(
401
+ entity.request["exception"]["code"]
402
+ )
423
403
  entity.request["nifi_attributes"]["raised_exc"] = True
424
404
 
425
405
  def sync_entities(self):
426
406
  if not self.lazy_sync:
427
407
  entities = self.out_entities
428
408
  else:
429
- entities = [
430
- e
431
- for e in self.out_entities
432
- if e.record["entity_id"] in self.nonlazy_sync_ids
433
- ]
409
+ entities = [e for e in self.out_entities if e.record["entity_id"] in self.nonlazy_sync_ids]
434
410
  if not entities:
435
411
  return
436
412
  reserved_fields = [
@@ -460,18 +436,16 @@ class NifiContextManager(object):
460
436
  for entity in entities:
461
437
  if entity.sync_params.get(NifiContextManager.SyncFlag.FETCH_RELATIONSHIPS):
462
438
  concept_name = entity.record["entity_type"]
463
- rels_to_fetch = entity.sync_params.get(
464
- NifiContextManager.SyncFlag.FETCH_RELATIONSHIPS, []
465
- )
439
+ rels_to_fetch = entity.sync_params.get(NifiContextManager.SyncFlag.FETCH_RELATIONSHIPS, [])
466
440
  for rel in rels_to_fetch:
467
441
  if rel not in fetch_relationships_entities:
468
442
  fetch_relationships_entities[rel] = []
469
443
  fetch_relationships_entities[rel].append(entity)
470
444
  if concept_name not in fetch_concept_relationships:
471
445
  fetch_concept_relationships[concept_name] = set()
472
- fetch_concept_relationships[concept_name] = fetch_concept_relationships[
473
- concept_name
474
- ].union(set(rels_to_fetch))
446
+ fetch_concept_relationships[concept_name] = fetch_concept_relationships[concept_name].union(
447
+ set(rels_to_fetch)
448
+ )
475
449
  for k in fetch_concept_relationships.keys():
476
450
  fetch_concept_relationships[k] = list(fetch_concept_relationships[k])
477
451
  # UPSERT ENTITIES
@@ -517,7 +491,7 @@ class NifiContextManager(object):
517
491
  file.request["is_temporary"] = False
518
492
  file.request["entity_timestamp"] = file.record["os_last_updated_at"]
519
493
  # FETCH RELATIONSHIPS
520
- '''
494
+ """
521
495
  if fetch_relationships_entities:
522
496
  relationship_mappings_info = relationship_mappings.sync_detailed(
523
497
  client=self.client
@@ -573,7 +547,7 @@ class NifiContextManager(object):
573
547
  child_rel.request["entity_timestamp"] = rel.get(
574
548
  "os_last_updated_at"
575
549
  )
576
- '''
550
+ """
577
551
  # CLEAN SYNC PARAMS
578
552
  for entity in entities:
579
553
  entity.sync_params = {}
@@ -582,15 +556,10 @@ class NifiContextManager(object):
582
556
  for entity in entities:
583
557
  fields = set()
584
558
 
585
- if (
586
- entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL)
587
- or entity.request["is_temporary"]
588
- ):
559
+ if entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL) or entity.request["is_temporary"]:
589
560
  fields = fields.union(set(list(entity.record.keys())))
590
561
 
591
- if entity.sync_params.get(
592
- NifiContextManager.SyncFlag.UPSERT_ENTITY_SPECIFIC_FIELDS
593
- ):
562
+ if entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_SPECIFIC_FIELDS):
594
563
  fields = fields.union(
595
564
  set(
596
565
  entity.sync_params.get(
@@ -601,9 +570,7 @@ class NifiContextManager(object):
601
570
  )
602
571
  )
603
572
  if fields:
604
- entities_to_upsert.append(
605
- (entity, [f for f in list(fields) if f not in reserved_fields])
606
- )
573
+ entities_to_upsert.append((entity, [f for f in list(fields) if f not in reserved_fields]))
607
574
 
608
575
  def __exit__(self, exc_type, exc_val, exc_tb):
609
576
  if exc_val is not None:
@@ -617,9 +584,7 @@ class NifiContextManager(object):
617
584
  for entity in entities:
618
585
  if isinstance(entity, NifiEntityProxy):
619
586
  children.extend(entity.children_entities)
620
- children.extend(
621
- _recursive_collect_proxies(entity.children_entities)
622
- )
587
+ children.extend(_recursive_collect_proxies(entity.children_entities))
623
588
  return children
624
589
 
625
590
  all_proxies = _recursive_collect_proxies(entities)
@@ -632,15 +597,21 @@ class NifiContextManager(object):
632
597
 
633
598
 
634
599
  class NifiEntity(object):
635
- def __init__(
636
- self, context, request, record, annotations, all_independent_uids, children=[], contents=None
637
- ):
600
+ def __init__(self, context, request, record, annotations, all_independent_uids, children=[], contents=None):
638
601
  self.context = context
639
602
  self.request = request
640
603
  self.record = record
641
604
  self.annotations = annotations
642
- assert self.record.get("os_entity_uid") and self.record.get("entity_id") and self.record["os_entity_uid"] == self.record["entity_id"]
643
- assert self.record.get("os_concept") and self.record.get("entity_type") and self.record["os_concept"] == self.record["entity_type"]
605
+ assert (
606
+ self.record.get("os_entity_uid")
607
+ and self.record.get("entity_id")
608
+ and self.record["os_entity_uid"] == self.record["entity_id"]
609
+ )
610
+ assert (
611
+ self.record.get("os_concept")
612
+ and self.record.get("entity_type")
613
+ and self.record["os_concept"] == self.record["entity_type"]
614
+ )
644
615
  if "entity_label" not in self.record:
645
616
  self.record["entity_label"] = self.label
646
617
  children = [c for c in children if isinstance(c, (str, dict))]
@@ -660,9 +631,7 @@ class NifiEntity(object):
660
631
  child_types = [c["entity_type"] for c in proxy_entity_children] + [
661
632
  c["record"]["entity_type"] for c in full_entity_children
662
633
  ]
663
- output_as_child = [False] * len(proxy_entity_children) + [True] * len(
664
- full_entity_children
665
- )
634
+ output_as_child = [False] * len(proxy_entity_children) + [True] * len(full_entity_children)
666
635
  output_as_independent = [uid in all_independent_uids for uid in child_uids]
667
636
  full_entity_children = [
668
637
  NifiEntity(
@@ -676,9 +645,7 @@ class NifiEntity(object):
676
645
  )
677
646
  for c in full_entity_children
678
647
  ]
679
- proxy_otm_children = [
680
- NifiOTMRelationshipProxy(**otm_child) for otm_child in proxy_otm_children
681
- ]
648
+ proxy_otm_children = [NifiOTMRelationshipProxy(**otm_child) for otm_child in proxy_otm_children]
682
649
  child_proxies = [None] * len(proxy_entity_children) + full_entity_children
683
650
  self.children = [
684
651
  NifiEntityProxy(
@@ -706,21 +673,18 @@ class NifiEntity(object):
706
673
 
707
674
  @property
708
675
  def sync_params(self):
709
- return {
710
- NifiContextManager.SyncFlag[k]: v
711
- for k, v in (self.request.get("sync_params") or {}).items()
712
- }
676
+ return {NifiContextManager.SyncFlag[k]: v for k, v in (self.request.get("sync_params") or {}).items()}
713
677
 
714
678
  @sync_params.setter
715
679
  def sync_params(self, new_params):
716
680
  self.request["sync_params"] = {
717
- (k.name if isinstance(k, NifiContextManager.SyncFlag) else k): v
718
- for k, v in new_params.items()
681
+ (k.name if isinstance(k, NifiContextManager.SyncFlag) else k): v for k, v in new_params.items()
719
682
  }
720
683
 
721
684
  @property
722
685
  def metadata(self):
723
686
  return self.annotations
687
+
724
688
  @metadata.setter
725
689
  def metadata(self, new_metadata):
726
690
  self.annotations = new_metadata
@@ -743,9 +707,7 @@ class NifiEntity(object):
743
707
  contents_pointer = deepcopy(self.request["contents_pointer"])
744
708
  ptr_location = contents_pointer.get("location")
745
709
  if ptr_location == "attachment" and not contents_pointer.get("pointer"):
746
- contents_pointer["pointer"] = (
747
- f"{self.record['os_workspace']}/{self.record['os_entity_uid']}"
748
- )
710
+ contents_pointer["pointer"] = f"{self.record['os_workspace']}/{self.record['os_entity_uid']}"
749
711
  return self.request["contents_pointer"]
750
712
 
751
713
  @contents_pointer.setter
@@ -765,9 +727,7 @@ class NifiEntity(object):
765
727
  return list(
766
728
  filter(
767
729
  lambda x: isinstance(x, NifiOTMRelationshipProxy)
768
- or is_child_concept_fn(
769
- x.entity_type, RELATIONSHIP_ENTITY_NAME, self.context.ontology
770
- ),
730
+ or is_child_concept_fn(x.entity_type, RELATIONSHIP_ENTITY_NAME, self.context.ontology),
771
731
  self.children,
772
732
  )
773
733
  )
@@ -786,18 +746,12 @@ class NifiEntity(object):
786
746
  )
787
747
  if (
788
748
  self.record.get("os_workspace")
789
- and (
790
- permissions.get(self.record.get("os_workspace")) or PermissionLevel.NONE
791
- )
792
- >= PermissionLevel.WRITE
749
+ and (permissions.get(self.record.get("os_workspace")) or PermissionLevel.NONE) >= PermissionLevel.WRITE
793
750
  ):
794
751
  return self.record["os_workspace"]
795
752
  elif (
796
753
  self.request.get("fallback_os_workspace")
797
- and (
798
- permissions.get(self.request.get("fallback_os_workspace"))
799
- or PermissionLevel.NONE
800
- )
754
+ and (permissions.get(self.request.get("fallback_os_workspace")) or PermissionLevel.NONE)
801
755
  >= PermissionLevel.WRITE
802
756
  ):
803
757
  return self.request["fallback_os_workspace"]
@@ -806,15 +760,8 @@ class NifiEntity(object):
806
760
 
807
761
  @property
808
762
  def label(self):
809
- if not self.context.ontology:
810
- return None
811
- label_fields = self.context.ontology["concepts"][
812
- self.record.get("os_concept") or self.record.get("entity_type")
813
- ]["labelKeys"]
814
- label_fields = [field for field in label_fields if field]
815
- label = " ".join(
816
- [(self.record.get(field) or "") for field in label_fields]
817
- ).strip()
763
+ label_keys = self.request["ontology"]["label_keys"]
764
+ label = " ".join([(self.record.get(field) or "") for field in label_keys]).strip()
818
765
  if not label:
819
766
  label = None
820
767
  return label
@@ -838,14 +785,10 @@ class NifiEntity(object):
838
785
  entity_type = None
839
786
  if isinstance(self, NifiEntityProxy):
840
787
  entity_type = self.entity_type
841
- return entity_type == type or is_child_concept_fn(
842
- entity_type, type, self.context.ontology
843
- )
788
+ return entity_type == type or is_child_concept_fn(entity_type, type, self.context.ontology)
844
789
  else:
845
790
  entity_type = self.record["entity_type"]
846
- return (
847
- entity_type == type or type in self.request["ontology_info"]["parents"]
848
- )
791
+ return entity_type == type or type in self.request["ontology_info"]["parents"]
849
792
 
850
793
  def is_fragmented(self) -> bool:
851
794
  return bool(self.request["config"].get("fragment", {}).get("fragments_stack"))
@@ -861,7 +804,7 @@ class NifiEntity(object):
861
804
  if _is_sub_fragment_recursive(value):
862
805
  return True
863
806
  return False
864
-
807
+
865
808
  if not self.is_fragmented():
866
809
  return True
867
810
  fragment = entity.request.get("config", {}).get("fragment", {})
@@ -888,10 +831,7 @@ class NifiEntity(object):
888
831
  else:
889
832
  proxy_entity_children.append(child)
890
833
  proxy_entity_children = list({c.uid: c for c in proxy_entity_children}.values())
891
- proxy_entity_children = [
892
- {"entity_id": c.uid, "entity_type": c.entity_type}
893
- for c in proxy_entity_children
894
- ]
834
+ proxy_entity_children = [{"entity_id": c.uid, "entity_type": c.entity_type} for c in proxy_entity_children]
895
835
  proxy_otm_children = list(
896
836
  {
897
837
  c.record["os_entity_uid_from"]
@@ -907,9 +847,7 @@ class NifiEntity(object):
907
847
  full_entity_children,
908
848
  key=lambda x: string_to_datetime(x.record.get("os_last_updated_at")),
909
849
  )
910
- full_entity_children = list(
911
- {c.uid: c.to_json() for c in full_entity_children}.values()
912
- )
850
+ full_entity_children = list({c.uid: c.to_json() for c in full_entity_children}.values())
913
851
  children = full_entity_children + proxy_entity_children + proxy_otm_children
914
852
  return {
915
853
  "request": self.request,
@@ -923,14 +861,21 @@ class NifiEntity(object):
923
861
  now_time = now()
924
862
  random_id = str(uuid.uuid4())
925
863
  username = self.jwt_data["username"]
864
+ if entity_type == self.record["entity_type"]:
865
+ ont_parents = self.request["ontology"]["parents"]
866
+ ont_relationships = self.request["ontology"]["relationships"]
867
+ ont_label_keys = self.request["ontology"]["label_keys"]
868
+ else:
869
+ ont_parents = self.context.ontology["concepts"][entity_type]["parents"]
870
+ ont_relationships = self.context.ontology["concepts"][entity_type]["relationships"]
871
+ ont_label_keys = get_label_keys(entity_type, self.context.ontology)
926
872
  child_request = {
927
873
  "jwt": self.request["jwt"],
928
874
  "ontology_name": self.request["ontology_name"],
929
875
  "ontology_info": {
930
- "parents": self.context.ontology["concepts"][entity_type]["parents"],
931
- "relationships": self.context.ontology["concepts"][entity_type][
932
- "relationships"
933
- ],
876
+ "parents": ont_parents,
877
+ "relationships": ont_relationships,
878
+ "label_keys": ont_label_keys,
934
879
  },
935
880
  "entity_timestamp": None,
936
881
  "sync_params": {},
@@ -986,11 +931,7 @@ class NifiEntity(object):
986
931
  ):
987
932
  return self._add_entity(
988
933
  os_relationship_workspace,
989
- (
990
- LOCAL_RELATIONSHIP_ENTITY_NAME
991
- if os_relationship_workspace
992
- else RELATIONSHIP_ENTITY_NAME
993
- ),
934
+ (LOCAL_RELATIONSHIP_ENTITY_NAME if os_relationship_workspace else RELATIONSHIP_ENTITY_NAME),
994
935
  {
995
936
  **relationship_fields,
996
937
  "os_entity_uid_from": os_entity_uid_from,
@@ -1072,9 +1013,7 @@ class NifiEntity(object):
1072
1013
  os_relationship_type,
1073
1014
  )
1074
1015
  child_entity._contents = file
1075
- child_entity.request["contents_pointer"] = (
1076
- NifiEntityModel.RequestModel.ContentsPointerModel(location="local")
1077
- )
1016
+ child_entity.request["contents_pointer"] = NifiEntityModel.RequestModel.ContentsPointerModel(location="local")
1078
1017
  return child_entity, child_rel
1079
1018
 
1080
1019
  def add_tag(self, os_workspace, name, group, order, color):
@@ -1087,21 +1026,20 @@ class NifiEntity(object):
1087
1026
  )
1088
1027
 
1089
1028
  def add_metadata(
1090
- self, json, merge_method: Callable[[Any, Any], Any], recurse: Union[bool, int] = False,
1029
+ self,
1030
+ json,
1031
+ merge_method: Callable[[Any, Any], Any],
1032
+ recurse: Union[bool, int] = False,
1091
1033
  ):
1092
1034
  if not self.metadata:
1093
1035
  self.metadata = {}
1094
- self.metadata = recursive_update_dict(
1095
- self.metadata, json, merge_method, recurse
1096
- )
1036
+ self.metadata = recursive_update_dict(self.metadata, json, merge_method, recurse)
1097
1037
 
1098
1038
  def propagate_metadata(self, to_entity, fields=None, merge_method=lambda _, v2: v2):
1099
1039
  metadata_to_propagate = deepcopy(self.metadata)
1100
1040
  if fields:
1101
1041
  metadata_to_propagate = {k: v for k, v in self.metadata if k in fields}
1102
- to_entity.metadata = recursive_update_dict(
1103
- to_entity.metadata, metadata_to_propagate, merge_method
1104
- )
1042
+ to_entity.metadata = recursive_update_dict(to_entity.metadata, metadata_to_propagate, merge_method)
1105
1043
 
1106
1044
 
1107
1045
  def more_recent_than(record_a, record_b):
@@ -1160,19 +1098,10 @@ class NifiRoute(Route):
1160
1098
  query_params = request.query_params
1161
1099
  processor_suffix = query_params["processor_suffix"]
1162
1100
  body = await request.json()
1163
- processor_name = (
1164
- "processor."
1165
- + self.processor_name
1166
- + "."
1167
- + op.replace("-", "_")
1168
- + "."
1169
- + processor_suffix
1170
- )
1101
+ processor_name = "processor." + self.processor_name + "." + op.replace("-", "_") + "." + processor_suffix
1171
1102
  if op not in self.endpoints.keys():
1172
1103
  raise StarletteHTTPException(401, f"Route {op} is forbidden for NiFi.")
1173
- task_id = await self.celery_executor.send_task(
1174
- self.endpoints[op], args=[body, processor_name]
1175
- )
1104
+ task_id = await self.celery_executor.send_task(self.endpoints[op], args=[body, processor_name])
1176
1105
  return task_id
1177
1106
 
1178
1107
  @staticmethod
@@ -69,10 +69,7 @@ def _sumy__luhn_call(summarizer, document):
69
69
 
70
70
 
71
71
  def get_nltk_tokenizer(language: str) -> Tokenizer:
72
- try:
73
- nltk.data.find("tokenizers/punkt")
74
- except LookupError:
75
- nltk.download("punkt")
72
+ nltk.data.find("tokenizers/punkt")
76
73
  return Tokenizer(language)
77
74
 
78
75
 
@@ -0,0 +1,13 @@
1
+ def is_child_concept(type, parent_type, ontology):
2
+ return type == parent_type or parent_type in ontology["concepts"][type]["parents"]
3
+
4
+ def get_label_keys(type, ontology):
5
+ parents = list(set(ontology["concepts"][type]["parents"]).add(type))
6
+ parents.reverse()
7
+ label_keys = {} # for guaranteed insertion order
8
+ for parent in parents:
9
+ for label_key in ontology["concepts"][parent]["labelKeys"]:
10
+ if not label_key:
11
+ continue
12
+ label_keys[label_key] = None
13
+ return list(label_keys.keys())
@@ -1,2 +0,0 @@
1
- def is_child_concept(type, parent_type, ontology):
2
- return type == parent_type or parent_type in ontology["concepts"][type]["parents"]