streamlit-octostar-utils 0.4.2.dev5__tar.gz → 0.4.2.dev7__tar.gz

{streamlit_octostar_utils-0.4.2.dev5 → streamlit_octostar_utils-0.4.2.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 0.4.2.dev5
+Version: 0.4.2.dev7
 Summary: 
 License: MIT
 License-File: LICENSE

{streamlit_octostar_utils-0.4.2.dev5 → streamlit_octostar_utils-0.4.2.dev7}/pyproject.toml

@@ -5,7 +5,7 @@ include = '\.pyi?$'
 
 [tool.poetry]
 name = "streamlit-octostar-utils"
-version = "0.4.2-dev.5"
+version = "0.4.2-dev.7"
 description = ""
 license = "MIT"
 authors = ["Octostar"]

{streamlit_octostar_utils-0.4.2.dev5 → streamlit_octostar_utils-0.4.2.dev7}/streamlit_octostar_utils/api_crafter/nifi.py

@@ -14,17 +14,18 @@ from fastapi import Request
 from fastapi.responses import JSONResponse
 from starlette.exceptions import HTTPException as StarletteHTTPException
 
-from octostar.utils.workspace import upsert_entities, write_file
+from octostar.utils.workspace import upsert_entities
 from octostar.utils.ontology import fetch_ontology_data
 from octostar.utils.workspace.permissions import get_permissions, PermissionLevel
-from octostar.client import make_client
 from octostar.utils.pipeline import update_processing_status
+from octostar.utils.workspace import write_attachment
+from octostar.client import make_client
 
 from ..core.dict import recursive_update_dict, travel_dict, jsondict_hash
 from ..core.timestamp import now, string_to_datetime
 from .fastapi import DefaultErrorRoute, Route
-from ..ontology.inheritance import is_child_concept as is_child_concept_fn, get_label_keys
-from .contents import Contents, WorkspaceAttachmentContents, TemporaryAttachmentContents, ContentsLocation
+from ..ontology.inheritance import is_child_concept as is_child_concept_fn, get_label, get_label_keys
+from .contents import Contents, WorkspaceAttachmentContents, TemporaryAttachmentContents
 
 
 class RelationshipName(BaseModel):
@@ -43,7 +44,11 @@ TAG_RELATIONSHIP = RelationshipName(name="has_tag", type="mtm")
 FRAGMENT_RELATIONSHIP = RelationshipName(name="is_fragment_of", type="otm")
 PREVIOUS_FRAGMENT_RELATIONSHIP = RelationshipName(name="has_previous_fragment", type="otm")
 NEXT_FRAGMENT_RELATIONSHIP = RelationshipName(name="has_next_fragment", type="otm")
-
+OS_RESERVED_FIELDS = [
+    "os_entity_uid", "entity_id", "entity_type", "os_concept",
+    "entity_label", "os_created_at", "os_created_by",
+    "os_last_updated_at", "os_last_updated_by",
+]
 
 class NifiProxyEntityModel(BaseModel):
     entity_id: str
@@ -73,7 +78,6 @@ class NifiEntityModel(BaseModel):
         nifi_attributes: dict = Field(default_factory=dict)
         config: dict = Field(default_factory=dict)
         metrics: dict = Field(default_factory=dict)
-        contents_pointer: Optional[dict] = None
         is_temporary: bool = False
         exception: dict = Field(default_factory=dict)
         last_processor_name: Optional[str] = None
@@ -429,35 +433,86 @@ class NifiContextManager(object):
             entities = [e for e in self.out_entities if e.record["entity_id"] in self.nonlazy_sync_ids]
         if not entities:
             return
-        reserved_fields = [
-            "os_entity_uid",
-            "entity_id",
-            "entity_type",
-            "os_concept",
-            "entity_label",
-            "os_created_at",
-            "os_created_by",
-            "os_last_updated_at",
-            "os_last_updated_by",
-        ]
-        entities_to_upsert = []
-        files_to_write = []
+
+        attachables_to_write = self._find_attachables_to_write(entities)
+        entities_to_upsert = self._find_entities_to_upsert(entities)
+        fetch_rel_entities, fetch_concept_rels = self._find_relationships_to_fetch(entities)
+
+        all_entities_to_modify = attachables_to_write + [e[0] for e in entities_to_upsert]
+        with self.reindex_lock(all_entities_to_modify):
+            self._sync_write_attachables(attachables_to_write)
+            self._sync_upsert_entities(entities_to_upsert)
+        self._sync_fetch_relationships(entities, fetch_rel_entities, fetch_concept_rels)
+
+        for entity in all_entities_to_modify:
+            entity.request["is_temporary"] = False
+            entity.request["entity_timestamp"] = entity.record["os_last_updated_at"]
+
+        for entity in entities:
+            entity.sync_params = {}
+
+    def _find_attachables_to_write(self, entities) -> list:
+        attachables_to_write = []
+        for entity in entities:
+            if entity.sync_params.get(NifiContextManager.SyncFlag.WRITE_CONTENTS):
+                if entity.contents:
+                    attachables_to_write.append(entity)
+        return attachables_to_write
+
+    def _sync_write_attachables(self, attachables_to_write):
+        for file in attachables_to_write:
+            if not file.contents:
+                continue
+            old_contents = file._contents
+            write_attachment.sync(
+                os_workspace=file.write_os_workspace,
+                os_entity_uid=file.record["os_entity_uid"],
+                entity_type=file.record["os_concept"],
+                filetype=file.record["os_item_content_type"],
+                file=file.contents,
+                client=self.client,
+            )
+            file._contents = WorkspaceAttachmentContents(
+                workspace_id=file.record['os_workspace'],
+                entity_id=file.record['os_entity_uid'],
+                client=self.client,
+                entity_type=file.record["os_concept"],
+                filetype=file.record["os_item_content_type"]
+            )
+            if isinstance(old_contents, TemporaryAttachmentContents):
+                try:
+                    old_contents.delete()
+                except Exception:
+                    pass
+
+    def _sync_upsert_entities(self, entities_to_upsert):
+        if not entities_to_upsert:
+            return
+        new_entities = upsert_entities.sync(
+            [entity.write_os_workspace for entity, _ in entities_to_upsert],
+            [
+                {
+                    "entity_type": entity.record["os_concept"],
+                    "os_entity_uid": entity.record["os_entity_uid"],
+                    "os_last_updated_at": entity.request["entity_timestamp"],
+                    "fields": {k: entity.record.get(k) for k in fields},
+                }
+                for entity, fields in entities_to_upsert
+            ],
+            client=self.client,
+        )
+        new_entities = {e["os_entity_uid"]: e for e in new_entities}
+        for entity, _ in entities_to_upsert:
+            entity.record = {**entity.record, **new_entities[entity.record["os_entity_uid"]]}
+            entity.record["entity_id"] = entity.record["os_entity_uid"]
+            entity.record["entity_type"] = entity.record["os_concept"]
+            entity.record["entity_label"] = entity.label
+            entity.request["entity_timestamp"] = entity.record["os_last_updated_at"]
+            entity.request["is_temporary"] = False
+
+    def _find_relationships_to_fetch(self, entities) -> dict:
         fetch_relationships_entities = {}
         fetch_concept_relationships = {}
-        # FIND FILES TO WRITE
-        for entity in entities:
-            if entity.is_child_concept("os_attachable"):
-                has_write_flag = entity.sync_params.get(NifiContextManager.SyncFlag.WRITE_CONTENTS)
-                is_temporary = entity.request.get("is_temporary")
-                if is_temporary:
-                    if entity._contents and not isinstance(entity._contents, WorkspaceAttachmentContents):
-                        files_to_write.append(entity)
-                elif has_write_flag:
-                    if entity.contents:
-                        files_to_write.append(entity)
-        # FIND ENTITIES TO UPSERT
-        self._find_entities_to_upsert(entities, entities_to_upsert, reserved_fields)
-        # FIND RELATIONSHIPS TO FETCH
         for entity in entities:
             if entity.sync_params.get(NifiContextManager.SyncFlag.FETCH_RELATIONSHIPS):
                 concept_name = entity.record["entity_type"]
@@ -473,142 +528,19 @@ class NifiContextManager(object):
                 )
         for k in fetch_concept_relationships.keys():
             fetch_concept_relationships[k] = list(fetch_concept_relationships[k])
-        # PROTECT AGAINST RE-INDEX OPS
-        all_entities_to_modify = (
-            files_to_write + [e[0] for e in entities_to_upsert]
-        )
-        with self.reindex_lock(all_entities_to_modify):
-            # WRITE FILES
-            if files_to_write:
-                for file in files_to_write:
-                    if not file.contents:
-                        continue
-                    old_contents = file._contents
-                    new_file_record = write_file.sync(
-                        file.write_os_workspace,
-                        "./" + file.record["os_item_name"],
-                        file.record["os_item_content_type"],
-                        file.contents,
-                        file.record["os_entity_uid"],
-                        file.record["os_parent_folder"],
-                        client=self.client,
-                    )
-                    file.record = {**file.record, **new_file_record}
-                    file.record["entity_id"] = file.record["os_entity_uid"]
-                    file.record["entity_type"] = file.record["os_concept"]
-                    file.record["entity_label"] = file.label
-                    file.request["is_temporary"] = False
-                    file.request["entity_timestamp"] = file.record["os_last_updated_at"]
-                    file._contents = WorkspaceAttachmentContents(
-                        workspace_id=file.record['os_workspace'],
-                        entity_id=file.record['os_entity_uid'],
-                        client=self.client,
-                        entity_type=file.record["os_concept"],
-                        filetype=file.record["os_item_content_type"]
-                    )
-                    file.request["contents_pointer"] = {
-                        "location": ContentsLocation.WORKSPACE_ATTACHMENT.value,
-                        "pointer": f"{file.record['os_workspace']}/{file.record['os_entity_uid']}",
-                        "entity_type": file.record["os_concept"],
-                        "filetype": file.record["os_item_content_type"]
-                    }
-                    if isinstance(old_contents, TemporaryAttachmentContents):
-                        try:
-                            old_contents.delete()
-                        except Exception:
-                            pass
-            # UPSERT ENTITIES
-            if entities_to_upsert:
-                new_entities = upsert_entities.sync(
-                    [entity.write_os_workspace for entity, _ in entities_to_upsert],
-                    [
-                        {
-                            "entity_type": entity.record["os_concept"],
-                            "os_entity_uid": entity.record["os_entity_uid"],
-                            "os_last_updated_at": entity.request["entity_timestamp"],
-                            "fields": {k: entity.record.get(k) for k in fields},
-                        }
-                        for entity, fields in entities_to_upsert
-                    ],
-                    client=self.client,
-                )
-                new_entities = {e["os_entity_uid"]: e for e in new_entities}
-                for entity, _ in entities_to_upsert:
-                    entity.record = {
-                        **entity.record,
-                        **new_entities[entity.record["os_entity_uid"]],
-                    }
-                    entity.record["entity_id"] = entity.record["os_entity_uid"]
-                    entity.record["entity_type"] = entity.record["os_concept"]
-                    entity.record["entity_label"] = entity.label
-                    entity.request["is_temporary"] = False
-                    entity.request["entity_timestamp"] = entity.record["os_last_updated_at"]
-        # FETCH RELATIONSHIPS
-        """
-        if fetch_relationships_entities:
-            relationship_mappings_info = relationship_mappings.sync_detailed(
-                client=self.client
-            ).parsed.data.to_dict()
-            expanded_entities, _ = safe_async_run(
-                expand_entities(
-                    [e.record for e in entities],
-                    self.ontology,
-                    relationship_mappings_info,
-                    self.client,
-                    fetch_concept_relationships,
-                )
-            )
-            expanded_entities = {e[0]["os_entity_uid"]: e for e in expanded_entities}
-            for entity in entities:
-                found_entities = expanded_entities.get(entity.record["os_entity_uid"])
-                if found_entities:
-                    for rel, target in found_entities[1]:
-                        is_mtm = bool(
-                            rel.get("os_entity_uid_from")
-                            and rel.get("os_entity_uid_to")
-                        )
-                        rel_type = "mtm" if is_mtm else "otm"
-                        child_entity, child_rel = entity.add_child_entity(
-                            target["os_workspace"],
-                            target["os_concept"],
-                            target,
-                            relationship=RelationshipName(name=rel["os_relationship_name"], type=rel_type),
-                        )
-                        child_entity.record = {**child_entity.record, **target}
-                        child_entity.record["entity_id"] = child_entity.record[
-                            "os_entity_uid"
-                        ]
-                        child_entity.uid = child_entity.record["os_entity_uid"]
-                        child_entity.request["is_temporary"] = False
-                        child_entity.request["entity_timestamp"] = target.get(
-                            "os_last_updated_at"
-                        )
-                        if not is_mtm:
-                            child_rel.record["os_entity_uid_to"] = child_entity.record[
-                                "os_entity_uid"
-                            ]
-                        else:
-                            child_rel.record = {**child_rel.record, **rel}
-                            child_rel.record["entity_id"] = child_rel.record[
-                                "os_entity_uid"
-                            ]
-                            child_rel.uid = child_rel.record["os_entity_uid"]
-                            child_rel.request["is_temporary"] = bool(
-                                rel.get("os_entity_uid")
-                            )
-                            child_rel.request["entity_timestamp"] = rel.get(
-                                "os_last_updated_at"
-                            )
-        """
-        # CLEAN SYNC PARAMS
-        for entity in entities:
-            entity.sync_params = {}
+        return fetch_relationships_entities, fetch_concept_relationships
+
+    def _sync_fetch_relationships(self, entities, fetch_relationships_entities, fetch_concept_relationships):
+        if not fetch_relationships_entities:
+            return
+        raise NotImplementedError("Relationship fetching is not yet implemented")
 
-    def _find_entities_to_upsert(self, entities, entities_to_upsert, reserved_fields) -> None:
+    def _find_entities_to_upsert(self, entities) -> list:
+        entities_to_upsert = []
         for entity in entities:
             fields = set()
 
-            if entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL) or entity.request["is_temporary"]:
+            if entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL):
                 fields = fields.union(set(list(entity.record.keys())))
 
             if entity.sync_params.get(NifiContextManager.SyncFlag.UPSERT_ENTITY_SPECIFIC_FIELDS):
@@ -622,7 +554,8 @@ class NifiContextManager(object):
                     )
                 )
             if fields:
-                entities_to_upsert.append((entity, [f for f in list(fields) if f not in reserved_fields]))
+                entities_to_upsert.append((entity, [f for f in list(fields) if f not in OS_RESERVED_FIELDS]))
+        return entities_to_upsert
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_val is not None:
@@ -743,15 +676,10 @@ class NifiEntity(object):
 
     @property
     def contents(self) -> Optional[Contents]:
-        if not self._contents:
-            contents_pointer = self.contents_pointer
-            if not contents_pointer:
-                return None
-            self._contents = Contents.from_locator(contents_pointer, client=self.context.client)
         return self._contents
 
     @contents.setter
-    def contents(self, new_contents: Optional[Union[Contents, bytes]]):
+    def contents(self, new_contents: Optional[Contents]):
         self._contents = new_contents
 
     @property
@@ -796,11 +724,17 @@ class NifiEntity(object):
 
     @property
     def label(self):
+        # dummy ontology from our cached data, compatible with get_label()
+        entity_type = self.record["entity_type"]
+        parents = self.request["ontology_info"]["parents"]
         label_keys = self.request["ontology_info"]["label_keys"]
-        label = " ".join([(self.record.get(field) or "") for field in label_keys]).strip()
-        if not label:
-            label = None
-        return label
+        ontology = {
+            "concepts": {
+                entity_type: {"parents": parents, "labelKeys": label_keys},
+                **{p: {"parents": [], "labelKeys": []} for p in parents}
+            }
+        }
+        return get_label(self.record, ontology)
 
     @property
     def filepath(self):
@@ -912,11 +846,12 @@ class NifiEntity(object):
                 "label_keys": ont_label_keys,
             },
             "entity_timestamp": None,
-            "sync_params": {},
+            "sync_params": {
+                NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL.name: True,
+            },
             "nifi_attributes": {},
             "config": deepcopy(self.request["config"]),
             "metrics": {},
-            "contents_pointer": None,
             "is_temporary": True,
             "exception": {},
             "last_processor_name": None,
@@ -991,7 +926,11 @@ class NifiEntity(object):
             )
             temp_contents.flush()
             child_entity._contents = temp_contents
-        child_entity.request["contents_pointer"] = child_entity._contents.to_locator()
+        child_entity.sync_params = recursive_update_dict(
+            child_entity.sync_params,
+            {NifiContextManager.SyncFlag.WRITE_CONTENTS: True},
+            lambda _, v2: v2
+        )
         return child_entity, child_rel
 
     def add_mtm_relationship(
@@ -1127,15 +1066,13 @@ class NifiEntity(object):
         fragment_uid = child_entity.record["os_entity_uid"]
         fragment_type = child_entity.record["os_concept"]
         if previous_fragment_uid:
-            prev_rel = previous_fragment_relationship or PREVIOUS_FRAGMENT_RELATIONSHIP
-            self._add_relationship(
+            prev_rel = self._add_relationship(
                 os_workspace, fragment_uid, fragment_type,
                 previous_fragment_uid, os_entity_type,
                 previous_fragment_relationship, {}, previous_fragment_relationship_uid,
             )
         if next_fragment_uid:
-            next_rel = next_fragment_relationship or NEXT_FRAGMENT_RELATIONSHIP
-            self._add_relationship(
+            next_rel = self._add_relationship(
                 os_workspace, fragment_uid, fragment_type,
                 next_fragment_uid, os_entity_type,
                 next_fragment_relationship, {}, next_fragment_relationship_uid,

{streamlit_octostar_utils-0.4.2.dev5 → streamlit_octostar_utils-0.4.2.dev7}/streamlit_octostar_utils/ontology/inheritance.py

@@ -13,3 +13,15 @@ def get_label_keys(type, ontology):
                 continue
             label_keys[label_key] = None
     return list(label_keys.keys())
+
+def get_label(record, ontology):
+    if record.get("os_materialized_label"):
+        return record.get("os_materialized_label")
+    if record.get("os_label"):
+        return record.get("os_label")
+    label_keys = get_label_keys(record["entity_type"], ontology)
+    fields = [record.get(field) for field in label_keys]
+    fields = [f for f in fields if f is not None]
+    fields = [str(f) for f in fields if f]
+    label = " ".join(fields)
+    return label or None