streamlit-octostar-utils 2.11a5__py3-none-any.whl → 2.11a8__py3-none-any.whl

streamlit_octostar_utils/api_crafter/fastapi.py

@@ -330,7 +330,7 @@ class DefaultErrorRoute:
         if len(message) > MAX_ERROR_MESSAGE_BYTES:
             message = message[-MAX_ERROR_MESSAGE_BYTES:]
         try:
-            tcbk = traceback.format_exception(exc)
+            tcbk = "\n".join(traceback.format_exception(exc))
             if len(tcbk) > MAX_ERROR_TRACEBACK_BYTES:
                 tcbk = tcbk[-MAX_ERROR_TRACEBACK_BYTES:]
         except:

streamlit_octostar_utils/api_crafter/nifi.py

@@ -23,7 +23,7 @@ from octostar.client import make_client
 from ..core.dict import recursive_update_dict, travel_dict, jsondict_hash
 from ..core.timestamp import now, string_to_datetime
 from .fastapi import DefaultErrorRoute, Route
-from ..ontology.inheritance import is_child_concept as is_child_concept_fn
+from ..ontology.inheritance import is_child_concept as is_child_concept_fn, get_label_keys
 from ..ontology.expand_entities import expand_entities
 
 RELATIONSHIP_ENTITY_NAME = "os_relationship"
@@ -69,6 +69,7 @@ class NifiEntityModel(BaseModel):
         class OntologyInfoModel(BaseModel):
             parents: List[str]
             relationships: List[str]
+            label_keys: List[str]
 
         class ContentsPointerModel(BaseModel):
             location: NifiContentsPointerLocationModel
@@ -806,15 +807,8 @@ class NifiEntity(object):
 
     @property
     def label(self):
-        if not self.context.ontology:
-            return None
-        label_fields = self.context.ontology["concepts"][
-            self.record.get("os_concept") or self.record.get("entity_type")
-        ]["labelKeys"]
-        label_fields = [field for field in label_fields if field]
-        label = " ".join(
-            [(self.record.get(field) or "") for field in label_fields]
-        ).strip()
+        label_keys = self.request["ontology_info"]["label_keys"]
+        label = " ".join([(self.record.get(field) or "") for field in label_keys]).strip()
         if not label:
             label = None
         return label
@@ -923,14 +917,21 @@ class NifiEntity(object):
         now_time = now()
         random_id = str(uuid.uuid4())
         username = self.jwt_data["username"]
+        if entity_type == self.record["entity_type"]:
+            ont_parents = self.request["ontology_info"]["parents"]
+            ont_relationships = self.request["ontology_info"]["relationships"]
+            ont_label_keys = self.request["ontology_info"]["label_keys"]
+        else:
+            ont_parents = self.context.ontology["concepts"][entity_type]["parents"]
+            ont_relationships = self.context.ontology["concepts"][entity_type]["relationships"]
+            ont_label_keys = get_label_keys(entity_type, self.context.ontology)
         child_request = {
             "jwt": self.request["jwt"],
             "ontology_name": self.request["ontology_name"],
             "ontology_info": {
-                "parents": self.context.ontology["concepts"][entity_type]["parents"],
-                "relationships": self.context.ontology["concepts"][entity_type][
-                    "relationships"
-                ],
+                "parents": ont_parents,
+                "relationships": ont_relationships,
+                "label_keys": ont_label_keys,
             },
             "entity_timestamp": None,
             "sync_params": {},

streamlit_octostar_utils/nlp/language.py

@@ -1,20 +1,41 @@
 import re
+from typing import Optional
+
 import py3langid as langid
-import iso639 as languages
+
+from iso639 import Lang, NonExistentLanguageError
+
+FLAIR_MODELS = {
+    "en": "flair/ner-english-large",
+    "es": "flair/ner-spanish-large",
+    "de": "flair/ner-german-large",
+    "nl": "flair/ner-dutch-large",
+    "multi": "flair/ner-multi",                     # English, German, French, Spanish
+    "multi-fast": "flair/ner-multi-fast",           # English, German, Dutch, Spanish
+}
+
+SPACY_MODELS = {
+    "en": "en_core_web_sm",
+    "es": "es_core_news_sm",
+    "fr": "fr_core_news_sm",
+    "de": "de_core_news_sm",
+    "it": "it_core_news_sm"
+}
 
 
 def alpha2_to_language(alpha2: str) -> str:
     if not alpha2:
-        return None
-    code = alpha2.strip().lower()
-    return languages.to_name(code)
+        raise ValueError("Language code must be a non-empty string.")
+    return Lang(alpha2).name
+
 
 def language_to_alpha2(language_name: str) -> str:
     if not language_name:
-        return None
-    name = language_name.strip().lower()
-    data = languages.find(name)
-    return data["iso639_1"]
+        raise ValueError("Language name must be a non-empty string.")
+
+    name = re.sub(r'\b\w+', lambda m: m.group(0).capitalize(), language_name)
+    return Lang(name).pt1
+
 
 def detect_language(text, min_confidence=None):
     detector = langid.langid.LanguageIdentifier.from_pickled_model(
@@ -23,6 +44,43 @@ def detect_language(text, min_confidence=None):
     detected_lang, confidence = detector.classify(text)
     if min_confidence and confidence < min_confidence:
         return None, confidence
-    detected_lang = re.sub("[^A-Za-z]", "", detected_lang).lower()
-    detected_lang = languages.to_name(detected_lang).lower()
+    detected_lang = alpha2_to_language(detected_lang)
     return detected_lang, confidence
+
+
+def is_language_available(language: Optional[str], type: str) -> bool:
+    if not language:
+        return False
+
+    try:
+        lang_code = language_to_alpha2(language)
+
+    except NonExistentLanguageError:
+        lang_code = language
+
+    match type:
+        case "spacy":
+            return SPACY_MODELS.get(lang_code, None) is not None
+
+        case "flair":
+            return FLAIR_MODELS.get(lang_code, None) is not None
+
+
+def load_language_model(language, type):
+    from flair.models import SequenceTagger
+    from spacy_download import load_spacy
+
+    match type:
+        case "spacy":
+            if is_language_available(language, "spacy"):
+                model_name = SPACY_MODELS.get(language_to_alpha2(language), SPACY_MODELS["en"])
+                return load_spacy(model_name)
+
+            raise Exception(f"SpaCy model for language '{language}' is not available.")
+
+        case "flair":
+            if is_language_available(language, "flair"):
+                model_name = FLAIR_MODELS.get(language, "flair/ner-multi")
+                return SequenceTagger.load(model_name)
+
+            raise Exception(f"Flair model for language '{language}' is not available.")

streamlit_octostar_utils/nlp/ner.py

@@ -20,7 +20,7 @@ from sumy.summarizers.lsa import LsaSummarizer
 from sumy.summarizers.luhn import LuhnSummarizer
 from sumy.utils import get_stop_words
 
-from .language import alpha2_to_language, language_to_alpha2
+from .language import alpha2_to_language
 
 BASE_ALLOWED_LABELS = ["PERSON", "ORG", "LOC", "NORP", "GPE", "PRODUCT", "DATE", "PHONE", "IP_ADDRESS", "EMAIL", "URL",
                        "CRYPTO", "IBAN", "CREDIT_CARD", "US_SSN", "US_DRIVER_LICENSE", "US_PASSPORT", "MEDICAL_LICENSE"]
@@ -67,38 +67,6 @@ BASE_TO_RECOGNIZER_EXPANSIONS = {
 
 BASE_TO_ONTONOTES_LABELMAP = {"PER": "PERSON"}
 
-FLAIR_MODELS = {
-    "en": "flair/ner-english-large",
-    "es": "flair/ner-spanish-large",
-    "de": "flair/ner-german-large",
-    "nl": "flair/ner-dutch-large",
-    "multi": "flair/ner-multi",                     # English, German, French, Spanish
-    "multi-fast": "flair/ner-multi-fast",           # English, German, Dutch, Spanish
-}
-
-SPACY_MODELS = {
-    "en": "en_core_web_sm",
-    "es": "es_core_news_sm",
-    "fr": "fr_core_news_sm",
-    "de": "de_core_news_sm",
-    "it": "it_core_news_sm"
-}
-
-def load_language_model(language, type):
-    from flair.models import SequenceTagger
-
-    model = None
-
-    match type:
-        case "spacy":
-            model = SPACY_MODELS.get(language_to_alpha2(language), SPACY_MODELS["en"])
-
-        case "flair":
-            model_name = FLAIR_MODELS.get(language, "flair/ner-multi")
-            model = SequenceTagger.load(model_name)
-
-    return model
-
 
 class FlairRecognizer(EntityRecognizer):
     ENTITIES = [
@@ -430,7 +398,7 @@ def analyze_dataframe_optimized(df: pd.DataFrame, analyzer: AnalyzerEngine, lang
 
 
 def compute_ner_presidio(
-        text,
+        text_or_df,
         language,
         analyzer,
         entities=None,
@@ -441,15 +409,15 @@ def compute_ner_presidio(
         batch_size=32,
         n_process=4
 ):
-    if isinstance(text, pd.DataFrame):
-        if len(text) >= 100:
-            return analyze_dataframe_optimized(text, analyzer, language, entities, score_threshold)
+    if isinstance(text_or_df, pd.DataFrame):
+        if len(text_or_df) >= 100:
+            return analyze_dataframe_optimized(text_or_df, analyzer, language, entities, score_threshold)
 
         else:
             texts = []
 
-            for col in text.columns:
-                for idx, value in text[col].dropna().items():
+            for col in text_or_df.columns:
+                for idx, value in text_or_df[col].dropna().items():
                     text_value = str(value).strip()
 
                     if text_value:
@@ -457,7 +425,8 @@ def compute_ner_presidio(
 
             text = "\n".join(texts)
 
-    elif isinstance(text, list):
+    elif isinstance(text_or_df, list):
+        text = text_or_df
         batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)
 
         results_generator = batch_analyzer.analyze_iterator(
@@ -566,7 +535,7 @@ def get_extractive_summary(text, language, max_chars, fast=False, with_scores=Fa
 
 
 def ner_pipe(
-        text,
+        text_or_df,
         language,
         model,
         engine_type="spacy",
@@ -589,9 +558,9 @@ def ner_pipe(
         model=model,
     )
 
-    if isinstance(text, pd.DataFrame):
+    if isinstance(text_or_df, pd.DataFrame):
         ner = compute_ner_presidio(
-            text,
+            text_or_df,
             language,
             analyzer,
             entities,
@@ -602,6 +571,8 @@ def ner_pipe(
             n_process=n_process
         )
     else:
+        text = text_or_df
+
         if compression_ratio == "auto":
             compression_ratio = max(1.0, len(text) / 15000) if fast else 1.0
 
@@ -640,8 +611,8 @@ def get_ner_handler(
     except LookupError:
         language = "en"
 
-    return lambda text, compression_ratio="auto", with_scores=False, with_comentions=True, with_context=True: ner_pipe(
-        text,
+    return lambda text_or_df, compression_ratio="auto", with_scores=False, with_comentions=True, with_context=True: ner_pipe(
+        text_or_df,
         language,
         model,
         engine_type,

streamlit_octostar_utils/ontology/inheritance.py

@@ -1,2 +1,15 @@
 def is_child_concept(type, parent_type, ontology):
     return type == parent_type or parent_type in ontology["concepts"][type]["parents"]
+
+def get_label_keys(type, ontology):
+    parents = set(ontology["concepts"][type]["parents"])
+    parents.add(type)
+    parents = list(parents)
+    parents.reverse()
+    label_keys = {} # for guaranteed insertion order
+    for parent in parents:
+        for label_key in ontology["concepts"][parent]["labelKeys"]:
+            if not label_key:
+                continue
+            label_keys[label_key] = None
+    return list(label_keys.keys())

{streamlit_octostar_utils-2.11a5.dist-info → streamlit_octostar_utils-2.11a8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 2.11a5
+Version: 2.11a8
 Summary: 
 License: MIT
 License-File: LICENSE

{streamlit_octostar_utils-2.11a5.dist-info → streamlit_octostar_utils-2.11a8.dist-info}/RECORD

@@ -1,8 +1,8 @@
 streamlit_octostar_utils/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 streamlit_octostar_utils/api_crafter/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 streamlit_octostar_utils/api_crafter/celery.py,sha256=BXOTGN9egdD75qf-PkccLGAoniilB9PZ_NRchFIjWdw,30051
-streamlit_octostar_utils/api_crafter/fastapi.py,sha256=2bktT5Mwjs9XixWcOqUKMoLM_cgKl-cqZDUa2Imf4xA,14357
-streamlit_octostar_utils/api_crafter/nifi.py,sha256=yFs1HXpSVfWpOC1aJnNahjPofGzZ8fpuqvChloqM4rQ,45541
+streamlit_octostar_utils/api_crafter/fastapi.py,sha256=RKQrStPzG1I1pxsPJvGs_DRrnjlMJbVmu9ObMF2LgZ0,14368
+streamlit_octostar_utils/api_crafter/nifi.py,sha256=6PSWIFKjv8nzlFGH9IFRI3VrYsISNjDIPyi1RvLJoKk,45810
 streamlit_octostar_utils/api_crafter/parser/__init__.py,sha256=YeYWF6sdQiCFV_RKNW2t9Vs6KJExE2pbXxWTe_DOayY,107
 streamlit_octostar_utils/api_crafter/parser/combine_fields.py,sha256=ddc44xkajw8MU0peAX_263DL7rPXbTKbHUjpOhRgvyU,8790
 streamlit_octostar_utils/api_crafter/parser/entities_parser.py,sha256=zOQoN-p1Gz6ZzxvoX4M1b4Fi3mfmQr5zaNUcp_8gCjw,30016
@@ -20,15 +20,15 @@ streamlit_octostar_utils/core/threading/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEkt
 streamlit_octostar_utils/core/threading/key_queue.py,sha256=7CJpj0gvZMQd8eC5wKQi3Ak5SQQ4zQ1OPTs_OP_kD20,2255
 streamlit_octostar_utils/core/timestamp.py,sha256=a3s4xfm1nctLzYsHOJxqoWIDTdbNY_yN1OByl8ahLc8,383
 streamlit_octostar_utils/nlp/__init__.py,sha256=BtlYDZK_xaEbc7Ju_7MznXbCVPZcdLn26xwR9qf_UhM,336
-streamlit_octostar_utils/nlp/language.py,sha256=l48rBoLLBpTZz40N2KWNSpAWc8smcWMtiiDXREhmLtE,926
-streamlit_octostar_utils/nlp/ner.py,sha256=LwnGbQHoT2mitroc0WjM2lVjtSUW7OUhqNmLsLMpNYQ,21196
+streamlit_octostar_utils/nlp/language.py,sha256=WEBhjr2UYgBGQnki0cY7d9kjp5RX5cYewUh57H6Om6o,2718
+streamlit_octostar_utils/nlp/ner.py,sha256=5swAuH7r9xZ7c48ApqZfLqidjdf6f2qxK52KLk7-9Cc,20406
 streamlit_octostar_utils/octostar/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 streamlit_octostar_utils/octostar/client.py,sha256=NUvHe9asd65g4-hJ4CuUvUns-9dNWes1XZRJlO9eAAc,1690
 streamlit_octostar_utils/octostar/context.py,sha256=TpucK48EbeVy4vDqKd9UULEtr1JOY-_4nBs-rXZzESw,212
 streamlit_octostar_utils/octostar/permissions.py,sha256=G5nZQLR-k-5_Xeto4nDTb32828Ga-SHm1mvSB9tz-t4,1565
 streamlit_octostar_utils/ontology/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 streamlit_octostar_utils/ontology/expand_entities.py,sha256=bBt32Dnts3VSzu13QQtPyfYe05IRodD9WfnhNTiBg_w,22749
-streamlit_octostar_utils/ontology/inheritance.py,sha256=oSd6xDAlmI7iYOv3VJ7t8CRN2zK7_Cln26YHS20qAqw,138
+streamlit_octostar_utils/ontology/inheritance.py,sha256=8GA2an1hbHfa6p993tIyfFLrewJHRUIFOw7dmvL8geU,583
 streamlit_octostar_utils/ontology/validation.py,sha256=0cXxEq8vQ63qxn4WianTioTcsmpsg4jEXVyI4R6x1gE,1051
 streamlit_octostar_utils/style/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 streamlit_octostar_utils/style/common.py,sha256=TKfjV9-sIoJChGM7Ewg3uPsz5sMmPxFwmc0o3L4D8Qo,1496
@@ -36,7 +36,7 @@ streamlit_octostar_utils/threading/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzp
 streamlit_octostar_utils/threading/async_task_manager.py,sha256=q7N6YZwUvIYMzkSHmsJNheNVCv93c03H6Hyg9uH8pvk,4747
 streamlit_octostar_utils/threading/session_callback_manager.py,sha256=LvZVP4g6tvKtYmI13f2j1sX_7hm61Groqp5xJine9_k,3973
 streamlit_octostar_utils/threading/session_state_hot_swapper.py,sha256=6eeCQI6A42hp4DmW2NQw2rbeR-k9N8DhfBKQdN_fbLU,811
-streamlit_octostar_utils-2.11a5.dist-info/METADATA,sha256=sa3ksvvDUHpMWd_szqcaFI_x9u7dVwc9Ctj1gcAyujg,2330
-streamlit_octostar_utils-2.11a5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-streamlit_octostar_utils-2.11a5.dist-info/licenses/LICENSE,sha256=dkwVPyV03fPHHtERnF6RnvRXcll__tud9gWca1RcgnQ,1073
-streamlit_octostar_utils-2.11a5.dist-info/RECORD,,
+streamlit_octostar_utils-2.11a8.dist-info/METADATA,sha256=uoeaIC6YWiZgZAa36JP45z-AyS-s-PP2uIpfkKjtm1k,2330
+streamlit_octostar_utils-2.11a8.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+streamlit_octostar_utils-2.11a8.dist-info/licenses/LICENSE,sha256=dkwVPyV03fPHHtERnF6RnvRXcll__tud9gWca1RcgnQ,1073
+streamlit_octostar_utils-2.11a8.dist-info/RECORD,,