streamlit-octostar-utils 0.5.4.dev5__tar.gz → 0.5.5.dev1__tar.gz

{streamlit_octostar_utils-0.5.4.dev5 → streamlit_octostar_utils-0.5.5.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 0.5.4.dev5
+Version: 0.5.5.dev1
 Summary: 
 License: MIT
 License-File: LICENSE
@@ -26,6 +26,7 @@ Requires-Dist: numpy (>=1.20.0)
 Requires-Dist: octostar-streamlit (>=0.1.25,<0.2.0)
 Requires-Dist: pottery (>=3.0.0,<4.0.0)
 Requires-Dist: presidio-analyzer (>=2.2.0,<3.0.0) ; extra == "nlp"
+Requires-Dist: psutil (>=5.9.0)
 Requires-Dist: py3langid (>=0.2.0,<0.3.0) ; extra == "nlp"
 Requires-Dist: pydantic (>=2.6.4,<3.0.0)
 Requires-Dist: python-multipart (>=0.0.9,<0.0.10)

{streamlit_octostar_utils-0.5.4.dev5 → streamlit_octostar_utils-0.5.5.dev1}/pyproject.toml

@@ -5,7 +5,7 @@ include = '\.pyi?$'
 
 [tool.poetry]
 name = "streamlit-octostar-utils"
-version = "0.5.4-dev.5"
+version = "0.5.5-dev.1"
 description = ""
 license = "MIT"
 authors = ["Octostar"]
@@ -37,6 +37,7 @@ numpy = ">=1.20.0"
 scipy = "^1.10.0"
 rapidfuzz = "^3.5.0"
 celery = "^5.3.0"
+psutil = ">=5.9.0"
 redis = ">=4.0.0,<5.0.0"
 pottery = "^3.0.0"
 slowapi = "^0.1.9"

{streamlit_octostar_utils-0.5.4.dev5 → streamlit_octostar_utils-0.5.5.dev1}/streamlit_octostar_utils/api_crafter/nifi.py

@@ -54,6 +54,144 @@ OS_RESERVED_FIELDS = [
 MAX_IN_MEMORY_SIZE_BYTES = 5_242_880
 
 
+class NifiPriority:
+    COMPONENT_WIDTHS = (10, 10, 1, 20)
+    SEPARATOR = "."
+    MAX_FRAGMENT_DEPTH = 9
+    BASE62_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+    BASE = len(BASE62_CHARS)
+    _BASE62_SET = frozenset(BASE62_CHARS)
+
+    @staticmethod
+    def _normalize_component(value, width):
+        if value is None:
+            return "z" * width
+        s = str(value)
+        invalid = set(s) - NifiPriority._BASE62_SET
+        if invalid:
+            raise ValueError(f"Invalid base62 character(s): {sorted(invalid)!r}")
+        if len(s) > width:
+            return "z" * width
+        return s.rjust(width, "0")
+
+    def __init__(self, op_reserved=None, user_prio=None, fragment_prio=0, entity_timestamp=0):
+        w = self.COMPONENT_WIDTHS
+        self.op_reserved = self._normalize_component(op_reserved, w[0])
+        self.user_prio = self._normalize_component(user_prio, w[1])
+        self.fragment_prio = int(fragment_prio)
+        self.entity_timestamp = int(entity_timestamp)
+
+    @staticmethod
+    def _encode_base62(value, width):
+        chars = NifiPriority.BASE62_CHARS
+        base = NifiPriority.BASE
+        if value < 0:
+            value = 0
+        if value == 0:
+            return "0" * width
+        result = []
+        v = value
+        while v > 0:
+            result.append(chars[v % base])
+            v //= base
+        result.reverse()
+        s = "".join(result)
+        if len(s) > width:
+            return chars[-1] * width
+        return s.rjust(width, "0")
+
+    @staticmethod
+    def _decode_base62(s):
+        chars = NifiPriority.BASE62_CHARS
+        result = 0
+        for c in s:
+            idx = chars.find(c)
+            if idx < 0:
+                raise ValueError(f"Invalid base62 character: {c!r}")
+            result = result * NifiPriority.BASE + idx
+        return result
+
+    @classmethod
+    def from_string(cls, s):
+        if not s or cls.SEPARATOR not in s:
+            return cls()
+        parts = s.split(cls.SEPARATOR)
+        if len(parts) != 4:
+            return cls()
+        try:
+            for part in parts:
+                if set(part) - cls._BASE62_SET:
+                    return cls()
+            return cls(
+                op_reserved=parts[0],
+                user_prio=parts[1],
+                fragment_prio=cls._decode_base62(parts[2]),
+                entity_timestamp=cls._decode_base62(parts[3]),
+            )
+        except (ValueError, TypeError):
+            return cls()
+
+    @classmethod
+    def from_dict(cls, d):
+        if not d:
+            return cls()
+        return cls(
+            op_reserved=d.get("op_reserved"),
+            user_prio=d.get("user_prio"),
+            fragment_prio=d.get("fragment_prio", 0),
+            entity_timestamp=d.get("entity_timestamp", 0),
+        )
+
+    @classmethod
+    def from_entity(cls, entity):
+        existing = entity.request.get("priority", {})
+        prio = cls.from_dict(existing)
+        prio.fragment_prio = cls.compute_fragment_depth(entity)
+        ts_str = entity.request.get("entity_timestamp")
+        if ts_str:
+            try:
+                dt = string_to_datetime(ts_str)
+                prio.entity_timestamp = int(dt.timestamp() * 1000)
+            except Exception:
+                prio.entity_timestamp = 0
+        else:
+            prio.entity_timestamp = 0
+        return prio
+
+    @staticmethod
+    def compute_fragment_depth(entity):
+        stack = entity.request.get("config", {}).get("fragment", {}).get("fragments_stack", [])
+        depth = 0
+        for key in stack:
+            try:
+                info = NifiFragmenter.get_fragment_info(entity, key)
+                if info.get("index", 0) != 0:
+                    depth += 1
+                else:
+                    break
+            except (KeyError, RuntimeError):
+                break
+        return min(depth, 9)
+
+    def to_string(self):
+        w = self.COMPONENT_WIDTHS
+        return self.SEPARATOR.join([
+            self.op_reserved,
+            self.user_prio,
+            self._encode_base62(self.fragment_prio, w[2]),
+            self._encode_base62(self.entity_timestamp, w[3]),
+        ])
+
+    def to_dict(self):
+        return {
+            "op_reserved": self.op_reserved,
+            "user_prio": self.user_prio,
+        }
+
+    def apply_to_entity(self, entity):
+        entity.request["priority"] = self.to_dict()
+
+
 class NifiProxyEntityModel(BaseModel):
     entity_id: str
     entity_type: str
@@ -86,6 +224,7 @@ class NifiEntityModel(BaseModel):
         exception: dict = Field(default_factory=dict)
         last_processor_name: Optional[str] = None
         fallback_os_workspace: Optional[str] = None
+        priority: dict = Field(default_factory=dict)
 
     class RecordModel(BaseModel):
         model_config = ConfigDict(extra="allow")
@@ -528,6 +667,8 @@ class NifiContextManager(object):
             key=lambda x: string_to_datetime(x.request.get("entity_timestamp")),
         )
         entities = list({e.record["entity_id"]: e for e in entities}.values())
+        for entity in entities:
+            NifiPriority.from_entity(entity).apply_to_entity(entity)
         entities = [
             (
                 jsondict_hash(NifiContextManager._config_get(entity, processor_name)),
@@ -602,6 +743,9 @@ class NifiContextManager(object):
         )
         self.out_entities = list({e.record["entity_id"]: e for e in all_entities}.values())
         self.sync_entities()
+        for entity in self.out_entities:
+            prio = NifiPriority.from_entity(entity)
+            entity.request["nifi_attributes"]["priority"] = prio.to_string()
         return [entity for entity in self.jsonify(self.out_entities)["content"]]
 
     def raise_exception(self, entity, exc):
@@ -1020,6 +1164,29 @@ class NifiEntity(object):
             options={"verify_signature": False},
         )
 
+    @property
+    def priority(self):
+        return NifiPriority.from_entity(self)
+
+    @priority.setter
+    def priority(self, value):
+        if isinstance(value, NifiPriority):
+            self.request["priority"] = value.to_dict()
+        elif isinstance(value, dict):
+            self.request["priority"] = value
+        else:
+            raise TypeError("priority must be a NifiPriority or dict")
+
+    def set_user_priority(self, value):
+        prio = self.priority
+        prio.user_prio = NifiPriority._normalize_component(value, NifiPriority.COMPONENT_WIDTHS[1])
+        self.priority = prio
+
+    def set_op_priority(self, value):
+        prio = self.priority
+        prio.op_reserved = NifiPriority._normalize_component(value, NifiPriority.COMPONENT_WIDTHS[0])
+        self.priority = prio
+
     def update_last_timestamp(self):
         self.record["os_last_updated_at"] = now()
 
@@ -1211,6 +1378,7 @@ class NifiEntity(object):
             "exception": {},
             "last_processor_name": None,
             "fallback_os_workspace": self.request["fallback_os_workspace"],
+            "priority": deepcopy(self.request.get("priority", {})),
         }
         child_entity = NifiEntity(
             self.context,