streamlit-octostar-utils 0.5.0.dev4__tar.gz → 0.5.0.dev6__tar.gz

{streamlit_octostar_utils-0.5.0.dev4 → streamlit_octostar_utils-0.5.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 0.5.0.dev4
+Version: 0.5.0.dev6
 Summary: 
 License: MIT
 License-File: LICENSE

{streamlit_octostar_utils-0.5.0.dev4 → streamlit_octostar_utils-0.5.0.dev6}/pyproject.toml

@@ -5,7 +5,7 @@ include = '\.pyi?$'
 
 [tool.poetry]
 name = "streamlit-octostar-utils"
-version = "0.5.0-dev.4"
+version = "0.5.0-dev.6"
 description = ""
 license = "MIT"
 authors = ["Octostar"]

{streamlit_octostar_utils-0.5.0.dev4 → streamlit_octostar_utils-0.5.0.dev6}/streamlit_octostar_utils/api_crafter/nifi.py

@@ -519,7 +519,7 @@ class NifiContextManager(object):
             )
         entities = sorted(
             entities,
-            key=lambda x: string_to_datetime(x.record.get("os_last_updated_at")),
+            key=lambda x: string_to_datetime(x.request.get("entity_timestamp")),
         )
         entities = list({e.record["entity_id"]: e for e in entities}.values())
         entities = [
@@ -592,7 +592,7 @@ class NifiContextManager(object):
             all_entities.extend(_process_entity(entity, processor_name))
         all_entities = sorted(
             all_entities,
-            key=lambda x: string_to_datetime(x.record.get("os_last_updated_at")),
+            key=lambda x: string_to_datetime(x.request.get("entity_timestamp")),
         )
         self.out_entities = list({e.record["entity_id"]: e for e in all_entities}.values())
         self.sync_entities()
@@ -616,32 +616,68 @@ class NifiContextManager(object):
     @contextmanager
     def reindex_lock(self, entities: List[Union[dict, "NifiEntity"]], timeout: int = 7200):
         if not entities:
-            yield
+            yield True
             return
-        entities = [e if isinstance(e, dict) else e.record for e in entities]
+
+        import logging
+        _lock_logger = logging.getLogger(__name__)
+
+        records = []
+        for e in entities:
+            if isinstance(e, dict):
+                records.append(
+                    (e, e.get("entity_timestamp"))
+                )
+            else:
+                records.append(
+                    (e.record, e.request.get("entity_timestamp") if e.request else None)
+                )
+
         long_expiry = (datetime.now(timezone.utc) + timedelta(seconds=timeout)).strftime("%Y-%m-%dT%H:%M:%SZ")
         statuses = [
             {
-                "entity_id": e["os_entity_uid"],
-                "entity_type": e["os_concept"],
-                "do_not_reindex_at": long_expiry,
+                "entity_id": rec["os_entity_uid"],
+                "entity_type": rec["os_concept"],
+                "do_not_update_before": ts,
+                "do_not_reindex_before": long_expiry,
             }
-            for e in entities
+            for rec, ts in records
+            if ts is not None
         ]
-        update_processing_status.sync(statuses, client=self.client)
+        if len(statuses) < len(records):
+            missing = [rec["os_entity_uid"] for rec, ts in records if ts is None]
+            _lock_logger.warning(
+                f"reindex_lock: {len(missing)} entities have no entity_timestamp, cannot set lock: {missing}"
+            )
+        if not statuses:
+            yield True
+            return
+
+        result = update_processing_status.sync(statuses, client=self.client)
+        skipped_ids = result.get("skipped_entity_ids", []) if isinstance(result, dict) else []
+        if skipped_ids:
+            _lock_logger.warning(
+                f"reindex_lock: {len(skipped_ids)} entities have a newer pipeline run, aborting sync: {skipped_ids}"
+            )
+            yield False
+            return
+
         try:
-            yield
+            yield True
         finally:
             short_expiry = (datetime.now(timezone.utc) + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ")
             statuses = [
                 {
-                    "entity_id": e["os_entity_uid"],
-                    "entity_type": e["os_concept"],
-                    "do_not_reindex_at": short_expiry,
+                    "entity_id": rec["os_entity_uid"],
+                    "entity_type": rec["os_concept"],
+                    "do_not_update_before": ts,
+                    "do_not_reindex_before": short_expiry,
                 }
-                for e in entities
+                for rec, ts in records
+                if ts is not None
             ]
-            update_processing_status.sync(statuses, client=self.client)
+            if statuses:
+                update_processing_status.sync(statuses, client=self.client)
 
     def sync_entities(self):
         if not self.lazy_sync:
@@ -656,14 +692,17 @@ class NifiContextManager(object):
         fetch_rel_entities, fetch_concept_rels = self._find_relationships_to_fetch(entities)
 
         all_entities_to_modify = attachables_to_write + [e[0] for e in entities_to_upsert]
-        with self.reindex_lock(all_entities_to_modify):
+        with self.reindex_lock(all_entities_to_modify) as locked:
+            if not locked:
+                return
             self._sync_write_attachables(attachables_to_write)
             self._sync_upsert_entities(entities_to_upsert)
         self._sync_fetch_relationships(entities, fetch_rel_entities, fetch_concept_rels)
 
+        now_ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
         for entity in all_entities_to_modify:
             entity.request["is_temporary"] = False
-            entity.request["entity_timestamp"] = entity.record["os_last_updated_at"]
+            entity.request["entity_timestamp"] = now_ts
 
         for entity in entities:
             entity.sync_params = {}
@@ -710,7 +749,7 @@ class NifiContextManager(object):
                 file.record["entity_id"] = file.record["os_entity_uid"]
                 file.record["entity_type"] = file.record["os_concept"]
                 file.record["entity_label"] = file.label
-            file.request["entity_timestamp"] = file.record["os_last_updated_at"]
+            file.request["entity_timestamp"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
             file.request["is_temporary"] = False
 
     def _sync_upsert_entities(self, entities_to_upsert):
@@ -722,7 +761,6 @@ class NifiContextManager(object):
                 {
                     "entity_type": entity.record["os_concept"],
                     "os_entity_uid": entity.record["os_entity_uid"],
-                    "os_last_updated_at": entity.request["entity_timestamp"],
                     "fields": {k: entity.record.get(k) for k in fields},
                 }
                 for entity, fields in entities_to_upsert
@@ -735,7 +773,7 @@ class NifiContextManager(object):
             entity.record["entity_id"] = entity.record["os_entity_uid"]
             entity.record["entity_type"] = entity.record["os_concept"]
             entity.record["entity_label"] = entity.label
-            entity.request["entity_timestamp"] = entity.record["os_last_updated_at"]
+            entity.request["entity_timestamp"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
             entity.request["is_temporary"] = False
 
     def _find_relationships_to_fetch(self, entities) -> dict:
@@ -1110,7 +1148,7 @@ class NifiEntity(object):
         proxy_otm_children = [c.record for c in proxy_otm_children]
         full_entity_children = sorted(
             full_entity_children,
-            key=lambda x: string_to_datetime(x.record.get("os_last_updated_at")),
+            key=lambda x: string_to_datetime(x.request.get("entity_timestamp")),
         )
         full_entity_children = list({c.uid: c.to_json() for c in full_entity_children}.values())
         children = full_entity_children + proxy_entity_children + proxy_otm_children
@@ -1143,7 +1181,7 @@ class NifiEntity(object):
                 "relationships": ont_relationships,
                 "label_keys": ont_label_keys,
             },
-            "entity_timestamp": None,
+            "entity_timestamp": self.request.get("entity_timestamp"),
             "sync_params": {
                 NifiContextManager.SyncFlag.UPSERT_ENTITY_ALL.name: True,
             },
@@ -1444,12 +1482,6 @@ class NifiEntity(object):
         to_entity.annotations = recursive_update_dict(to_entity.annotations, annotations_to_propagate, merge_method)
 
 
-def more_recent_than(record_a, record_b):
-    return string_to_datetime(record_a.get("os_last_updated_at")) >= string_to_datetime(
-        record_b.get("os_last_updated_at")
-    )
-
-
 class NifiRoute(Route):
     def __init__(self, app, tasks_routes, processor_name, celery_executor, router=None):
         self.app = app