streamlit-octostar-utils 0.2.10__py3-none-any.whl → 0.2.12a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- streamlit_octostar_utils/api_crafter/nifi.py +86 -157
- streamlit_octostar_utils/ontology/inheritance.py +13 -0
- {streamlit_octostar_utils-0.2.10.dist-info → streamlit_octostar_utils-0.2.12a1.dist-info}/METADATA +1 -1
- {streamlit_octostar_utils-0.2.10.dist-info → streamlit_octostar_utils-0.2.12a1.dist-info}/RECORD +6 -6
- {streamlit_octostar_utils-0.2.10.dist-info → streamlit_octostar_utils-0.2.12a1.dist-info}/WHEEL +1 -1
- {streamlit_octostar_utils-0.2.10.dist-info → streamlit_octostar_utils-0.2.12a1.dist-info}/licenses/LICENSE +0 -0
@@ -23,7 +23,7 @@ from octostar.client import make_client
|
|
23
23
|
from ..core.dict import recursive_update_dict, travel_dict, jsondict_hash
|
24
24
|
from ..core.timestamp import now, string_to_datetime
|
25
25
|
from .fastapi import DefaultErrorRoute, Route
|
26
|
-
from ..ontology.inheritance import is_child_concept as is_child_concept_fn
|
26
|
+
from ..ontology.inheritance import is_child_concept as is_child_concept_fn, get_label_keys
|
27
27
|
from ..ontology.expand_entities import expand_entities
|
28
28
|
|
29
29
|
RELATIONSHIP_ENTITY_NAME = "os_relationship"
|
@@ -69,6 +69,7 @@ class NifiEntityModel(BaseModel):
|
|
69
69
|
class OntologyInfoModel(BaseModel):
|
70
70
|
parents: List[str]
|
71
71
|
relationships: List[str]
|
72
|
+
label_keys: List[str]
|
72
73
|
|
73
74
|
class ContentsPointerModel(BaseModel):
|
74
75
|
location: NifiContentsPointerLocationModel
|
@@ -162,16 +163,12 @@ class NifiEntityProxy(object):
|
|
162
163
|
if child_entity.uid == uid_to_search:
|
163
164
|
found_entity = child_entity
|
164
165
|
else:
|
165
|
-
found_entity = _recursive_search_expanded_proxy(
|
166
|
-
child_entity._proxy, uid_to_search
|
167
|
-
)
|
166
|
+
found_entity = _recursive_search_expanded_proxy(child_entity._proxy, uid_to_search)
|
168
167
|
if found_entity:
|
169
168
|
return found_entity
|
170
169
|
|
171
170
|
if not self._proxy:
|
172
|
-
main_entities = itertools.chain(
|
173
|
-
*[b.entities for b in self.context.in_batches]
|
174
|
-
)
|
171
|
+
main_entities = itertools.chain(*[b.entities for b in self.context.in_batches])
|
175
172
|
main_entities = {e.record["entity_id"]: e for e in main_entities}
|
176
173
|
if main_entities.get(self.uid):
|
177
174
|
self._proxy = main_entities.get(self.uid)
|
@@ -182,9 +179,7 @@ class NifiEntityProxy(object):
|
|
182
179
|
self._proxy = found_entity._proxy
|
183
180
|
return self._proxy
|
184
181
|
## TODO: Try to get the entity from the database with query_ontology()
|
185
|
-
raise AttributeError(
|
186
|
-
f"Cannot find children with UUID {self.uid}! It may exist in the database?"
|
187
|
-
)
|
182
|
+
raise AttributeError(f"Cannot find children with UUID {self.uid}! It may exist in the database?")
|
188
183
|
|
189
184
|
def __getattr__(self, name):
|
190
185
|
if name in self.__dict__:
|
@@ -218,28 +213,24 @@ class NifiFragmenter(object):
|
|
218
213
|
raise ValueError("Must have at least 2 entities for fragmentation")
|
219
214
|
identifier = str(uuid.uuid4())
|
220
215
|
for i, entity in enumerate(fragments):
|
221
|
-
travel_dict(
|
222
|
-
|
223
|
-
)
|
216
|
+
travel_dict(entity.request["nifi_attributes"], fragmenter_keylist.split("."), "w")(
|
217
|
+
{"identifier": identifier, "count": count, "index": i}
|
218
|
+
)
|
224
219
|
if "fragment" not in entity.request["config"]:
|
225
220
|
entity.request["config"]["fragment"] = {}
|
226
221
|
if "fragments_stack" not in entity.request["config"]["fragment"]:
|
227
222
|
entity.request["config"]["fragment"]["fragments_stack"] = []
|
228
|
-
entity.request["config"]["fragment"]["fragments_stack"].insert(
|
229
|
-
|
223
|
+
entity.request["config"]["fragment"]["fragments_stack"].insert(0, fragmenter_keylist)
|
224
|
+
entity.request["nifi_attributes"]["fragments_stack"] = entity.request["config"]["fragment"][
|
225
|
+
"fragments_stack"
|
226
|
+
]
|
227
|
+
travel_dict(entity.request["config"]["fragment"], fragmenter_keylist.split("."), "w")(
|
228
|
+
{"identifier": identifier, "count": count, "index": i}
|
230
229
|
)
|
231
|
-
entity.request["nifi_attributes"]["fragments_stack"] = entity.request[
|
232
|
-
"config"
|
233
|
-
]["fragment"]["fragments_stack"]
|
234
|
-
travel_dict(
|
235
|
-
entity.request["config"]["fragment"], fragmenter_keylist.split("."), "w"
|
236
|
-
)({"identifier": identifier, "count": count, "index": i})
|
237
230
|
|
238
231
|
def push_defragment_strategy(fragment, defragmenter_config):
|
239
232
|
pointer = fragment.request["config"]
|
240
|
-
last_fragmenter_keylist = fragment.request["config"]["fragment"][
|
241
|
-
"fragments_stack"
|
242
|
-
][0]
|
233
|
+
last_fragmenter_keylist = fragment.request["config"]["fragment"]["fragments_stack"][0]
|
243
234
|
for k in ("fragment." + last_fragmenter_keylist).split("."):
|
244
235
|
if not pointer.get(k):
|
245
236
|
pointer[k] = {}
|
@@ -258,7 +249,7 @@ class NifiEntityBatch(object):
|
|
258
249
|
|
259
250
|
class NifiContextManager(object):
|
260
251
|
HEADLESS_PROCESSOR_NAME = "headless"
|
261
|
-
|
252
|
+
|
262
253
|
class SyncFlag(Enum):
|
263
254
|
UPSERT_ENTITY_ALL = 0 # bool
|
264
255
|
UPSERT_ENTITY_SPECIFIC_FIELDS = 1 # 'fields': list of record fields
|
@@ -279,9 +270,7 @@ class NifiContextManager(object):
|
|
279
270
|
@property
|
280
271
|
def ontology(self):
|
281
272
|
if not self._ontology:
|
282
|
-
self._ontology = fetch_ontology_data.sync(
|
283
|
-
ontology_name=self.ontology_name, client=self.client
|
284
|
-
)
|
273
|
+
self._ontology = fetch_ontology_data.sync(ontology_name=self.ontology_name, client=self.client)
|
285
274
|
return self._ontology
|
286
275
|
|
287
276
|
def _config_get(entity, keylist):
|
@@ -350,9 +339,7 @@ class NifiContextManager(object):
|
|
350
339
|
return self
|
351
340
|
|
352
341
|
def get_workspaces_permissions(self, workspace_ids):
|
353
|
-
permissions_to_fetch = list(
|
354
|
-
set(workspace_ids).difference(set(list(self.permissions.keys())))
|
355
|
-
)
|
342
|
+
permissions_to_fetch = list(set(workspace_ids).difference(set(list(self.permissions.keys()))))
|
356
343
|
if permissions_to_fetch:
|
357
344
|
permissions = get_permissions.sync(permissions_to_fetch, client=self.client)
|
358
345
|
self.permissions.update(permissions)
|
@@ -382,18 +369,13 @@ class NifiContextManager(object):
|
|
382
369
|
entities.append(entity)
|
383
370
|
for child_entity in entity.children_entities:
|
384
371
|
if not child_entity.drop_on_output:
|
385
|
-
if
|
386
|
-
child_entity.output_as_independent
|
387
|
-
or child_entity.output_as_child
|
388
|
-
):
|
372
|
+
if child_entity.output_as_independent or child_entity.output_as_child:
|
389
373
|
if processor_name != NifiContextManager.HEADLESS_PROCESSOR_NAME:
|
390
374
|
child_entity.request["last_processor_name"] = processor_name
|
391
375
|
if child_entity.output_as_independent:
|
392
376
|
if not child_entity._proxy:
|
393
377
|
child_entity.fetch_proxy()
|
394
|
-
entities.extend(
|
395
|
-
_process_entity(child_entity._proxy, processor_name)
|
396
|
-
)
|
378
|
+
entities.extend(_process_entity(child_entity._proxy, processor_name))
|
397
379
|
return entities
|
398
380
|
|
399
381
|
entities = itertools.chain(*[b.entities for b in entity_batches])
|
@@ -404,9 +386,7 @@ class NifiContextManager(object):
|
|
404
386
|
all_entities,
|
405
387
|
key=lambda x: string_to_datetime(x.record.get("os_last_updated_at")),
|
406
388
|
)
|
407
|
-
self.out_entities = list(
|
408
|
-
{e.record["entity_id"]: e for e in all_entities}.values()
|
409
|
-
)
|
389
|
+
self.out_entities = list({e.record["entity_id"]: e for e in all_entities}.values())
|
410
390
|
self.sync_entities()
|
411
391
|
return [entity for entity in self.jsonify(self.out_entities)["content"]]
|
412
392
|
|
@@ -414,23 +394,19 @@ class NifiContextManager(object):
|
|
414
394
|
error_response = DefaultErrorRoute.format_error(exc)
|
415
395
|
entity.request["exception"]["code"] = error_response.status_code
|
416
396
|
entity.request["exception"]["body"] = json.loads(error_response.body)["message"]
|
417
|
-
travel_dict(
|
418
|
-
entity.request["
|
419
|
-
)
|
420
|
-
travel_dict(
|
421
|
-
entity.request["
|
422
|
-
)
|
397
|
+
travel_dict(entity.request["nifi_attributes"], ["invokehttp", "response", "body"], "w")(
|
398
|
+
entity.request["exception"]["body"]
|
399
|
+
)
|
400
|
+
travel_dict(entity.request["nifi_attributes"], ["invokehttp", "response", "code"], "w")(
|
401
|
+
entity.request["exception"]["code"]
|
402
|
+
)
|
423
403
|
entity.request["nifi_attributes"]["raised_exc"] = True
|
424
404
|
|
425
405
|
def sync_entities(self):
|
426
406
|
if not self.lazy_sync:
|
427
407
|
entities = self.out_entities
|
428
408
|
else:
|
429
|
-
entities = [
|
430
|
-
e
|
431
|
-
for e in self.out_entities
|
432
|
-
if e.record["entity_id"] in self.nonlazy_sync_ids
|
433
|
-
]
|
409
|
+
entities = [e for e in self.out_entities if e.record["entity_id"] in self.nonlazy_sync_ids]
|
434
410
|
if not entities:
|
435
411
|
return
|
436
412
|
reserved_fields = [
|
@@ -460,18 +436,16 @@ class NifiContextManager(object):
|
|
460
436
|
for entity in entities:
|
461
437
|
if entity.sync_params.get(NifiContextManager.SyncFlag.FETCH_RELATIONSHIPS):
|
462
438
|
concept_name = entity.record["entity_type"]
|
463
|
-
rels_to_fetch = entity.sync_params.get(
|
464
|
-
NifiContextManager.SyncFlag.FETCH_RELATIONSHIPS, []
|
465
|
-
)
|
439
|
+
rels_to_fetch = entity.sync_params.get(NifiContextManager.SyncFlag.FETCH_RELATIONSHIPS, [])
|
466
440
|
for rel in rels_to_fetch:
|
467
441
|
if rel not in fetch_relationships_entities:
|
468
442
|
fetch_relationships_entities[rel] = []
|
469
443
|
fetch_relationships_entities[rel].append(entity)
|
470
444
|
if concept_name not in fetch_concept_relationships:
|
471
445
|
fetch_concept_relationships[concept_name] = set()
|
472
|
-
fetch_concept_relationships[concept_name] = fetch_concept_relationships[
|
473
|
-
|
474
|
-
|
446
|
+
fetch_concept_relationships[concept_name] = fetch_concept_relationships[concept_name].union(
|
447
|
+
set(rels_to_fetch)
|
448
|
+
)
|
475
449
|
for k in fetch_concept_relationships.keys():
|
476
450
|
fetch_concept_relationships[k] = list(fetch_concept_relationships[k])
|
477
451
|
# UPSERT ENTITIES
|
@@ -517,7 +491,7 @@ class NifiContextManager(object):
|
|
517
491
|
file.request["is_temporary"] = False
|
518
492
|
file.request["entity_timestamp"] = file.record["os_last_updated_at"]
|
519
493
|
# FETCH RELATIONSHIPS
|
520
|
-
|
494
|
+
"""
|
521
495
|
if fetch_relationships_entities:
|
522
496
|
relationship_mappings_info = relationship_mappings.sync_detailed(
|
523
497
|
client=self.client
|
@@ -573,7 +547,7 @@ class NifiContextManager(object):
|
|
573
547
|
child_rel.request["entity_timestamp"] = rel.get(
|
574
548
|
"os_last_updated_at"
|
575
549
|
)
|
576
|
-
|
550
|
+
"""
|
577
551
|
# CLEAN SYNC PARAMS
|
578
552
|
for entity in entities:
|
579
553
|
entity.sync_params = {}
|
@@ -582,15 +556,10 @@ class NifiContextManager(object):
|
|
582
556
|
for entity in entities:
|
583
557
|
fields = set()
|
584
558
|
|
585
|
-
if (
|
586
|
-
entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL)
|
587
|
-
or entity.request["is_temporary"]
|
588
|
-
):
|
559
|
+
if entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL) or entity.request["is_temporary"]:
|
589
560
|
fields = fields.union(set(list(entity.record.keys())))
|
590
561
|
|
591
|
-
if entity.sync_params.get(
|
592
|
-
NifiContextManager.SyncFlag.UPSERT_ENTITY_SPECIFIC_FIELDS
|
593
|
-
):
|
562
|
+
if entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_SPECIFIC_FIELDS):
|
594
563
|
fields = fields.union(
|
595
564
|
set(
|
596
565
|
entity.sync_params.get(
|
@@ -601,9 +570,7 @@ class NifiContextManager(object):
|
|
601
570
|
)
|
602
571
|
)
|
603
572
|
if fields:
|
604
|
-
entities_to_upsert.append(
|
605
|
-
(entity, [f for f in list(fields) if f not in reserved_fields])
|
606
|
-
)
|
573
|
+
entities_to_upsert.append((entity, [f for f in list(fields) if f not in reserved_fields]))
|
607
574
|
|
608
575
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
609
576
|
if exc_val is not None:
|
@@ -617,9 +584,7 @@ class NifiContextManager(object):
|
|
617
584
|
for entity in entities:
|
618
585
|
if isinstance(entity, NifiEntityProxy):
|
619
586
|
children.extend(entity.children_entities)
|
620
|
-
children.extend(
|
621
|
-
_recursive_collect_proxies(entity.children_entities)
|
622
|
-
)
|
587
|
+
children.extend(_recursive_collect_proxies(entity.children_entities))
|
623
588
|
return children
|
624
589
|
|
625
590
|
all_proxies = _recursive_collect_proxies(entities)
|
@@ -632,15 +597,21 @@ class NifiContextManager(object):
|
|
632
597
|
|
633
598
|
|
634
599
|
class NifiEntity(object):
|
635
|
-
def __init__(
|
636
|
-
self, context, request, record, annotations, all_independent_uids, children=[], contents=None
|
637
|
-
):
|
600
|
+
def __init__(self, context, request, record, annotations, all_independent_uids, children=[], contents=None):
|
638
601
|
self.context = context
|
639
602
|
self.request = request
|
640
603
|
self.record = record
|
641
604
|
self.annotations = annotations
|
642
|
-
assert
|
643
|
-
|
605
|
+
assert (
|
606
|
+
self.record.get("os_entity_uid")
|
607
|
+
and self.record.get("entity_id")
|
608
|
+
and self.record["os_entity_uid"] == self.record["entity_id"]
|
609
|
+
)
|
610
|
+
assert (
|
611
|
+
self.record.get("os_concept")
|
612
|
+
and self.record.get("entity_type")
|
613
|
+
and self.record["os_concept"] == self.record["entity_type"]
|
614
|
+
)
|
644
615
|
if "entity_label" not in self.record:
|
645
616
|
self.record["entity_label"] = self.label
|
646
617
|
children = [c for c in children if isinstance(c, (str, dict))]
|
@@ -660,9 +631,7 @@ class NifiEntity(object):
|
|
660
631
|
child_types = [c["entity_type"] for c in proxy_entity_children] + [
|
661
632
|
c["record"]["entity_type"] for c in full_entity_children
|
662
633
|
]
|
663
|
-
output_as_child = [False] * len(proxy_entity_children) + [True] * len(
|
664
|
-
full_entity_children
|
665
|
-
)
|
634
|
+
output_as_child = [False] * len(proxy_entity_children) + [True] * len(full_entity_children)
|
666
635
|
output_as_independent = [uid in all_independent_uids for uid in child_uids]
|
667
636
|
full_entity_children = [
|
668
637
|
NifiEntity(
|
@@ -676,9 +645,7 @@ class NifiEntity(object):
|
|
676
645
|
)
|
677
646
|
for c in full_entity_children
|
678
647
|
]
|
679
|
-
proxy_otm_children = [
|
680
|
-
NifiOTMRelationshipProxy(**otm_child) for otm_child in proxy_otm_children
|
681
|
-
]
|
648
|
+
proxy_otm_children = [NifiOTMRelationshipProxy(**otm_child) for otm_child in proxy_otm_children]
|
682
649
|
child_proxies = [None] * len(proxy_entity_children) + full_entity_children
|
683
650
|
self.children = [
|
684
651
|
NifiEntityProxy(
|
@@ -706,21 +673,18 @@ class NifiEntity(object):
|
|
706
673
|
|
707
674
|
@property
|
708
675
|
def sync_params(self):
|
709
|
-
return {
|
710
|
-
NifiContextManager.SyncFlag[k]: v
|
711
|
-
for k, v in (self.request.get("sync_params") or {}).items()
|
712
|
-
}
|
676
|
+
return {NifiContextManager.SyncFlag[k]: v for k, v in (self.request.get("sync_params") or {}).items()}
|
713
677
|
|
714
678
|
@sync_params.setter
|
715
679
|
def sync_params(self, new_params):
|
716
680
|
self.request["sync_params"] = {
|
717
|
-
(k.name if isinstance(k, NifiContextManager.SyncFlag) else k): v
|
718
|
-
for k, v in new_params.items()
|
681
|
+
(k.name if isinstance(k, NifiContextManager.SyncFlag) else k): v for k, v in new_params.items()
|
719
682
|
}
|
720
683
|
|
721
684
|
@property
|
722
685
|
def metadata(self):
|
723
686
|
return self.annotations
|
687
|
+
|
724
688
|
@metadata.setter
|
725
689
|
def metadata(self, new_metadata):
|
726
690
|
self.annotations = new_metadata
|
@@ -743,9 +707,7 @@ class NifiEntity(object):
|
|
743
707
|
contents_pointer = deepcopy(self.request["contents_pointer"])
|
744
708
|
ptr_location = contents_pointer.get("location")
|
745
709
|
if ptr_location == "attachment" and not contents_pointer.get("pointer"):
|
746
|
-
contents_pointer["pointer"] =
|
747
|
-
f"{self.record['os_workspace']}/{self.record['os_entity_uid']}"
|
748
|
-
)
|
710
|
+
contents_pointer["pointer"] = f"{self.record['os_workspace']}/{self.record['os_entity_uid']}"
|
749
711
|
return self.request["contents_pointer"]
|
750
712
|
|
751
713
|
@contents_pointer.setter
|
@@ -765,9 +727,7 @@ class NifiEntity(object):
|
|
765
727
|
return list(
|
766
728
|
filter(
|
767
729
|
lambda x: isinstance(x, NifiOTMRelationshipProxy)
|
768
|
-
or is_child_concept_fn(
|
769
|
-
x.entity_type, RELATIONSHIP_ENTITY_NAME, self.context.ontology
|
770
|
-
),
|
730
|
+
or is_child_concept_fn(x.entity_type, RELATIONSHIP_ENTITY_NAME, self.context.ontology),
|
771
731
|
self.children,
|
772
732
|
)
|
773
733
|
)
|
@@ -786,18 +746,12 @@ class NifiEntity(object):
|
|
786
746
|
)
|
787
747
|
if (
|
788
748
|
self.record.get("os_workspace")
|
789
|
-
and (
|
790
|
-
permissions.get(self.record.get("os_workspace")) or PermissionLevel.NONE
|
791
|
-
)
|
792
|
-
>= PermissionLevel.WRITE
|
749
|
+
and (permissions.get(self.record.get("os_workspace")) or PermissionLevel.NONE) >= PermissionLevel.WRITE
|
793
750
|
):
|
794
751
|
return self.record["os_workspace"]
|
795
752
|
elif (
|
796
753
|
self.request.get("fallback_os_workspace")
|
797
|
-
and (
|
798
|
-
permissions.get(self.request.get("fallback_os_workspace"))
|
799
|
-
or PermissionLevel.NONE
|
800
|
-
)
|
754
|
+
and (permissions.get(self.request.get("fallback_os_workspace")) or PermissionLevel.NONE)
|
801
755
|
>= PermissionLevel.WRITE
|
802
756
|
):
|
803
757
|
return self.request["fallback_os_workspace"]
|
@@ -806,15 +760,8 @@ class NifiEntity(object):
|
|
806
760
|
|
807
761
|
@property
|
808
762
|
def label(self):
|
809
|
-
|
810
|
-
|
811
|
-
label_fields = self.context.ontology["concepts"][
|
812
|
-
self.record.get("os_concept") or self.record.get("entity_type")
|
813
|
-
]["labelKeys"]
|
814
|
-
label_fields = [field for field in label_fields if field]
|
815
|
-
label = " ".join(
|
816
|
-
[(self.record.get(field) or "") for field in label_fields]
|
817
|
-
).strip()
|
763
|
+
label_keys = self.request["ontology"]["label_keys"]
|
764
|
+
label = " ".join([(self.record.get(field) or "") for field in label_keys]).strip()
|
818
765
|
if not label:
|
819
766
|
label = None
|
820
767
|
return label
|
@@ -838,14 +785,10 @@ class NifiEntity(object):
|
|
838
785
|
entity_type = None
|
839
786
|
if isinstance(self, NifiEntityProxy):
|
840
787
|
entity_type = self.entity_type
|
841
|
-
return entity_type == type or is_child_concept_fn(
|
842
|
-
entity_type, type, self.context.ontology
|
843
|
-
)
|
788
|
+
return entity_type == type or is_child_concept_fn(entity_type, type, self.context.ontology)
|
844
789
|
else:
|
845
790
|
entity_type = self.record["entity_type"]
|
846
|
-
return
|
847
|
-
entity_type == type or type in self.request["ontology_info"]["parents"]
|
848
|
-
)
|
791
|
+
return entity_type == type or type in self.request["ontology_info"]["parents"]
|
849
792
|
|
850
793
|
def is_fragmented(self) -> bool:
|
851
794
|
return bool(self.request["config"].get("fragment", {}).get("fragments_stack"))
|
@@ -861,7 +804,7 @@ class NifiEntity(object):
|
|
861
804
|
if _is_sub_fragment_recursive(value):
|
862
805
|
return True
|
863
806
|
return False
|
864
|
-
|
807
|
+
|
865
808
|
if not self.is_fragmented():
|
866
809
|
return True
|
867
810
|
fragment = entity.request.get("config", {}).get("fragment", {})
|
@@ -888,10 +831,7 @@ class NifiEntity(object):
|
|
888
831
|
else:
|
889
832
|
proxy_entity_children.append(child)
|
890
833
|
proxy_entity_children = list({c.uid: c for c in proxy_entity_children}.values())
|
891
|
-
proxy_entity_children = [
|
892
|
-
{"entity_id": c.uid, "entity_type": c.entity_type}
|
893
|
-
for c in proxy_entity_children
|
894
|
-
]
|
834
|
+
proxy_entity_children = [{"entity_id": c.uid, "entity_type": c.entity_type} for c in proxy_entity_children]
|
895
835
|
proxy_otm_children = list(
|
896
836
|
{
|
897
837
|
c.record["os_entity_uid_from"]
|
@@ -907,9 +847,7 @@ class NifiEntity(object):
|
|
907
847
|
full_entity_children,
|
908
848
|
key=lambda x: string_to_datetime(x.record.get("os_last_updated_at")),
|
909
849
|
)
|
910
|
-
full_entity_children = list(
|
911
|
-
{c.uid: c.to_json() for c in full_entity_children}.values()
|
912
|
-
)
|
850
|
+
full_entity_children = list({c.uid: c.to_json() for c in full_entity_children}.values())
|
913
851
|
children = full_entity_children + proxy_entity_children + proxy_otm_children
|
914
852
|
return {
|
915
853
|
"request": self.request,
|
@@ -923,14 +861,21 @@ class NifiEntity(object):
|
|
923
861
|
now_time = now()
|
924
862
|
random_id = str(uuid.uuid4())
|
925
863
|
username = self.jwt_data["username"]
|
864
|
+
if entity_type == self.record["entity_type"]:
|
865
|
+
ont_parents = self.request["ontology"]["parents"]
|
866
|
+
ont_relationships = self.request["ontology"]["relationships"]
|
867
|
+
ont_label_keys = self.request["ontology"]["label_keys"]
|
868
|
+
else:
|
869
|
+
ont_parents = self.context.ontology["concepts"][entity_type]["parents"]
|
870
|
+
ont_relationships = self.context.ontology["concepts"][entity_type]["relationships"]
|
871
|
+
ont_label_keys = get_label_keys(entity_type, self.context.ontology)
|
926
872
|
child_request = {
|
927
873
|
"jwt": self.request["jwt"],
|
928
874
|
"ontology_name": self.request["ontology_name"],
|
929
875
|
"ontology_info": {
|
930
|
-
"parents":
|
931
|
-
"relationships":
|
932
|
-
|
933
|
-
],
|
876
|
+
"parents": ont_parents,
|
877
|
+
"relationships": ont_relationships,
|
878
|
+
"label_keys": ont_label_keys,
|
934
879
|
},
|
935
880
|
"entity_timestamp": None,
|
936
881
|
"sync_params": {},
|
@@ -986,11 +931,7 @@ class NifiEntity(object):
|
|
986
931
|
):
|
987
932
|
return self._add_entity(
|
988
933
|
os_relationship_workspace,
|
989
|
-
(
|
990
|
-
LOCAL_RELATIONSHIP_ENTITY_NAME
|
991
|
-
if os_relationship_workspace
|
992
|
-
else RELATIONSHIP_ENTITY_NAME
|
993
|
-
),
|
934
|
+
(LOCAL_RELATIONSHIP_ENTITY_NAME if os_relationship_workspace else RELATIONSHIP_ENTITY_NAME),
|
994
935
|
{
|
995
936
|
**relationship_fields,
|
996
937
|
"os_entity_uid_from": os_entity_uid_from,
|
@@ -1072,9 +1013,7 @@ class NifiEntity(object):
|
|
1072
1013
|
os_relationship_type,
|
1073
1014
|
)
|
1074
1015
|
child_entity._contents = file
|
1075
|
-
child_entity.request["contents_pointer"] = (
|
1076
|
-
NifiEntityModel.RequestModel.ContentsPointerModel(location="local")
|
1077
|
-
)
|
1016
|
+
child_entity.request["contents_pointer"] = NifiEntityModel.RequestModel.ContentsPointerModel(location="local")
|
1078
1017
|
return child_entity, child_rel
|
1079
1018
|
|
1080
1019
|
def add_tag(self, os_workspace, name, group, order, color):
|
@@ -1087,21 +1026,20 @@ class NifiEntity(object):
|
|
1087
1026
|
)
|
1088
1027
|
|
1089
1028
|
def add_metadata(
|
1090
|
-
self,
|
1029
|
+
self,
|
1030
|
+
json,
|
1031
|
+
merge_method: Callable[[Any, Any], Any],
|
1032
|
+
recurse: Union[bool, int] = False,
|
1091
1033
|
):
|
1092
1034
|
if not self.metadata:
|
1093
1035
|
self.metadata = {}
|
1094
|
-
self.metadata = recursive_update_dict(
|
1095
|
-
self.metadata, json, merge_method, recurse
|
1096
|
-
)
|
1036
|
+
self.metadata = recursive_update_dict(self.metadata, json, merge_method, recurse)
|
1097
1037
|
|
1098
1038
|
def propagate_metadata(self, to_entity, fields=None, merge_method=lambda _, v2: v2):
|
1099
1039
|
metadata_to_propagate = deepcopy(self.metadata)
|
1100
1040
|
if fields:
|
1101
1041
|
metadata_to_propagate = {k: v for k, v in self.metadata if k in fields}
|
1102
|
-
to_entity.metadata = recursive_update_dict(
|
1103
|
-
to_entity.metadata, metadata_to_propagate, merge_method
|
1104
|
-
)
|
1042
|
+
to_entity.metadata = recursive_update_dict(to_entity.metadata, metadata_to_propagate, merge_method)
|
1105
1043
|
|
1106
1044
|
|
1107
1045
|
def more_recent_than(record_a, record_b):
|
@@ -1160,19 +1098,10 @@ class NifiRoute(Route):
|
|
1160
1098
|
query_params = request.query_params
|
1161
1099
|
processor_suffix = query_params["processor_suffix"]
|
1162
1100
|
body = await request.json()
|
1163
|
-
processor_name = (
|
1164
|
-
"processor."
|
1165
|
-
+ self.processor_name
|
1166
|
-
+ "."
|
1167
|
-
+ op.replace("-", "_")
|
1168
|
-
+ "."
|
1169
|
-
+ processor_suffix
|
1170
|
-
)
|
1101
|
+
processor_name = "processor." + self.processor_name + "." + op.replace("-", "_") + "." + processor_suffix
|
1171
1102
|
if op not in self.endpoints.keys():
|
1172
1103
|
raise StarletteHTTPException(401, f"Route {op} is forbidden for NiFi.")
|
1173
|
-
task_id = await self.celery_executor.send_task(
|
1174
|
-
self.endpoints[op], args=[body, processor_name]
|
1175
|
-
)
|
1104
|
+
task_id = await self.celery_executor.send_task(self.endpoints[op], args=[body, processor_name])
|
1176
1105
|
return task_id
|
1177
1106
|
|
1178
1107
|
@staticmethod
|
@@ -1,2 +1,15 @@
|
|
1
1
|
def is_child_concept(type, parent_type, ontology):
|
2
2
|
return type == parent_type or parent_type in ontology["concepts"][type]["parents"]
|
3
|
+
|
4
|
+
def get_label_keys(type, ontology):
|
5
|
+
parents = set(ontology["concepts"][type]["parents"])
|
6
|
+
parents.add(type)
|
7
|
+
parents = list(parents)
|
8
|
+
parents.reverse()
|
9
|
+
label_keys = {} # for guaranteed insertion order
|
10
|
+
for parent in parents:
|
11
|
+
for label_key in ontology["concepts"][parent]["labelKeys"]:
|
12
|
+
if not label_key:
|
13
|
+
continue
|
14
|
+
label_keys[label_key] = None
|
15
|
+
return list(label_keys.keys())
|
{streamlit_octostar_utils-0.2.10.dist-info → streamlit_octostar_utils-0.2.12a1.dist-info}/RECORD
RENAMED
@@ -2,7 +2,7 @@ streamlit_octostar_utils/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-Y
|
|
2
2
|
streamlit_octostar_utils/api_crafter/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
3
3
|
streamlit_octostar_utils/api_crafter/celery.py,sha256=BXOTGN9egdD75qf-PkccLGAoniilB9PZ_NRchFIjWdw,30051
|
4
4
|
streamlit_octostar_utils/api_crafter/fastapi.py,sha256=RKQrStPzG1I1pxsPJvGs_DRrnjlMJbVmu9ObMF2LgZ0,14368
|
5
|
-
streamlit_octostar_utils/api_crafter/nifi.py,sha256=
|
5
|
+
streamlit_octostar_utils/api_crafter/nifi.py,sha256=IPWP5KasbQxx4G3Vwkh8gBMq0Lkz86EeZNzBJLU_vqg,44607
|
6
6
|
streamlit_octostar_utils/api_crafter/parser/__init__.py,sha256=YeYWF6sdQiCFV_RKNW2t9Vs6KJExE2pbXxWTe_DOayY,107
|
7
7
|
streamlit_octostar_utils/api_crafter/parser/combine_fields.py,sha256=ddc44xkajw8MU0peAX_263DL7rPXbTKbHUjpOhRgvyU,8790
|
8
8
|
streamlit_octostar_utils/api_crafter/parser/entities_parser.py,sha256=zOQoN-p1Gz6ZzxvoX4M1b4Fi3mfmQr5zaNUcp_8gCjw,30016
|
@@ -28,7 +28,7 @@ streamlit_octostar_utils/octostar/context.py,sha256=TpucK48EbeVy4vDqKd9UULEtr1JO
|
|
28
28
|
streamlit_octostar_utils/octostar/permissions.py,sha256=G5nZQLR-k-5_Xeto4nDTb32828Ga-SHm1mvSB9tz-t4,1565
|
29
29
|
streamlit_octostar_utils/ontology/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
30
30
|
streamlit_octostar_utils/ontology/expand_entities.py,sha256=bBt32Dnts3VSzu13QQtPyfYe05IRodD9WfnhNTiBg_w,22749
|
31
|
-
streamlit_octostar_utils/ontology/inheritance.py,sha256=
|
31
|
+
streamlit_octostar_utils/ontology/inheritance.py,sha256=T5W94NdW5GIigEwiFCBXcL57H8qQbu2gUBy4gj1FlwU,582
|
32
32
|
streamlit_octostar_utils/ontology/validation.py,sha256=0cXxEq8vQ63qxn4WianTioTcsmpsg4jEXVyI4R6x1gE,1051
|
33
33
|
streamlit_octostar_utils/style/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
34
34
|
streamlit_octostar_utils/style/common.py,sha256=TKfjV9-sIoJChGM7Ewg3uPsz5sMmPxFwmc0o3L4D8Qo,1496
|
@@ -36,7 +36,7 @@ streamlit_octostar_utils/threading/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzp
|
|
36
36
|
streamlit_octostar_utils/threading/async_task_manager.py,sha256=q7N6YZwUvIYMzkSHmsJNheNVCv93c03H6Hyg9uH8pvk,4747
|
37
37
|
streamlit_octostar_utils/threading/session_callback_manager.py,sha256=LvZVP4g6tvKtYmI13f2j1sX_7hm61Groqp5xJine9_k,3973
|
38
38
|
streamlit_octostar_utils/threading/session_state_hot_swapper.py,sha256=6eeCQI6A42hp4DmW2NQw2rbeR-k9N8DhfBKQdN_fbLU,811
|
39
|
-
streamlit_octostar_utils-0.2.
|
40
|
-
streamlit_octostar_utils-0.2.
|
41
|
-
streamlit_octostar_utils-0.2.
|
42
|
-
streamlit_octostar_utils-0.2.
|
39
|
+
streamlit_octostar_utils-0.2.12a1.dist-info/METADATA,sha256=DrFzRGdV9en7ueT3zAINd7gfmqoaqldNmuwDJxYRoUU,2332
|
40
|
+
streamlit_octostar_utils-0.2.12a1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
41
|
+
streamlit_octostar_utils-0.2.12a1.dist-info/licenses/LICENSE,sha256=dkwVPyV03fPHHtERnF6RnvRXcll__tud9gWca1RcgnQ,1073
|
42
|
+
streamlit_octostar_utils-0.2.12a1.dist-info/RECORD,,
|
File without changes
|