streamlit-octostar-utils 2.11a4__py3-none-any.whl → 2.11a5__py3-none-any.whl

streamlit_octostar_utils/nlp/language.py

@@ -26,30 +26,3 @@ def detect_language(text, min_confidence=None):
     detected_lang = re.sub("[^A-Za-z]", "", detected_lang).lower()
     detected_lang = languages.to_name(detected_lang).lower()
     return detected_lang, confidence
-
-FLAIR_MODELS = {
-    "en": "flair/ner-english-large",
-    "es": "flair/ner-spanish-large",
-    "de": "flair/ner-german-large",
-    "nl": "flair/ner-dutch-large",
-    "multi": "flair/ner-multi",
-    "multi-fast": "flair/ner-multi-fast",
-}
-
-SPACY_MODELS = {
-    "en": 'en_core_web_sm',
-}
-
-def load_language_model(language, type):
-    from flair.models import SequenceTagger
-    from spacy_download import load_spacy
-
-    model = None
-    match type:
-        case "spacy":
-            model_name = SPACY_MODELS.get(language, SPACY_MODELS["en"])
-            model = load_spacy(model_name)
-        case "flair":
-            model_name = FLAIR_MODELS.get(language, "flair/ner-multi")
-            model = SequenceTagger.load(model_name)
-    return model

streamlit_octostar_utils/nlp/ner.py

@@ -20,7 +20,7 @@ from sumy.summarizers.lsa import LsaSummarizer
 from sumy.summarizers.luhn import LuhnSummarizer
 from sumy.utils import get_stop_words
 
-from .language import alpha2_to_language
+from .language import alpha2_to_language, language_to_alpha2
 
 BASE_ALLOWED_LABELS = ["PERSON", "ORG", "LOC", "NORP", "GPE", "PRODUCT", "DATE", "PHONE", "IP_ADDRESS", "EMAIL", "URL",
                        "CRYPTO", "IBAN", "CREDIT_CARD", "US_SSN", "US_DRIVER_LICENSE", "US_PASSPORT", "MEDICAL_LICENSE"]
@@ -67,6 +67,38 @@ BASE_TO_RECOGNIZER_EXPANSIONS = {
 
 BASE_TO_ONTONOTES_LABELMAP = {"PER": "PERSON"}
 
+FLAIR_MODELS = {
+    "en": "flair/ner-english-large",
+    "es": "flair/ner-spanish-large",
+    "de": "flair/ner-german-large",
+    "nl": "flair/ner-dutch-large",
+    "multi": "flair/ner-multi",                     # English, German, French, Spanish
+    "multi-fast": "flair/ner-multi-fast",           # English, German, Dutch, Spanish
+}
+
+SPACY_MODELS = {
+    "en": "en_core_web_sm",
+    "es": "es_core_news_sm",
+    "fr": "fr_core_news_sm",
+    "de": "de_core_news_sm",
+    "it": "it_core_news_sm"
+}
+
+def load_language_model(language, type):
+    from flair.models import SequenceTagger
+
+    model = None
+
+    match type:
+        case "spacy":
+            model = SPACY_MODELS.get(language_to_alpha2(language), SPACY_MODELS["en"])
+
+        case "flair":
+            model_name = FLAIR_MODELS.get(language, "flair/ner-multi")
+            model = SequenceTagger.load(model_name)
+
+    return model
+
 
 class FlairRecognizer(EntityRecognizer):
     ENTITIES = [
@@ -83,15 +115,6 @@ class FlairRecognizer(EntityRecognizer):
         ({"ORG"}, {"ORG", "ORGANIZATION"}),
     ]
 
-    MODEL_LANGUAGES = {
-        "en": "flair/ner-english-large",
-        "es": "flair/ner-spanish-large",
-        "de": "flair/ner-german-large",
-        "nl": "flair/ner-dutch-large",
-        "multi": "flair/ner-multi",
-        "multi-fast": "flair/ner-multi-fast",
-    }
-
     PRESIDIO_EQUIVALENCES = {
         "PER": "PERSON",
         "LOC": "LOC",
@@ -549,6 +572,7 @@ def ner_pipe(
         engine_type="spacy",
         fast=False,
         compression_ratio="auto",
+        with_scores=False,
         with_comentions=True,
         with_context=True,
         entities=None,
@@ -556,6 +580,9 @@ def ner_pipe(
         batch_size=32,
         n_process=4
 ):
+    if with_scores:
+        raise NotImplementedError("with_scores functionality is not implemented yet")
+
     analyzer = build_presidio_analyzer(
         language=language,
         engine_type=engine_type,
@@ -613,13 +640,14 @@ def get_ner_handler(
     except LookupError:
         language = "en"
 
-    return lambda text, compression_ratio="auto", with_comentions=True, with_context=True: ner_pipe(
+    return lambda text, compression_ratio="auto", with_scores=False, with_comentions=True, with_context=True: ner_pipe(
         text,
         language,
         model,
         engine_type,
         fast,
         compression_ratio,
+        with_scores,
         with_comentions,
         with_context,
         entities,

{streamlit_octostar_utils-2.11a4.dist-info → streamlit_octostar_utils-2.11a5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: streamlit-octostar-utils
-Version: 2.11a4
+Version: 2.11a5
 Summary: 
 License: MIT
 License-File: LICENSE

{streamlit_octostar_utils-2.11a4.dist-info → streamlit_octostar_utils-2.11a5.dist-info}/RECORD

@@ -20,8 +20,8 @@ streamlit_octostar_utils/core/threading/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEkt
 streamlit_octostar_utils/core/threading/key_queue.py,sha256=7CJpj0gvZMQd8eC5wKQi3Ak5SQQ4zQ1OPTs_OP_kD20,2255
 streamlit_octostar_utils/core/timestamp.py,sha256=a3s4xfm1nctLzYsHOJxqoWIDTdbNY_yN1OByl8ahLc8,383
 streamlit_octostar_utils/nlp/__init__.py,sha256=BtlYDZK_xaEbc7Ju_7MznXbCVPZcdLn26xwR9qf_UhM,336
-streamlit_octostar_utils/nlp/language.py,sha256=2d8Wq8wTuo_ehjZekuoe3bgJD52ieEiZKDUPdKdOxZ0,1699
-streamlit_octostar_utils/nlp/ner.py,sha256=BP32wkZUNaKVIyzREEAgluPfwiISmNE4uITg7g1p0PM,20381
+streamlit_octostar_utils/nlp/language.py,sha256=l48rBoLLBpTZz40N2KWNSpAWc8smcWMtiiDXREhmLtE,926
+streamlit_octostar_utils/nlp/ner.py,sha256=LwnGbQHoT2mitroc0WjM2lVjtSUW7OUhqNmLsLMpNYQ,21196
 streamlit_octostar_utils/octostar/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 streamlit_octostar_utils/octostar/client.py,sha256=NUvHe9asd65g4-hJ4CuUvUns-9dNWes1XZRJlO9eAAc,1690
 streamlit_octostar_utils/octostar/context.py,sha256=TpucK48EbeVy4vDqKd9UULEtr1JOY-_4nBs-rXZzESw,212
@@ -36,7 +36,7 @@ streamlit_octostar_utils/threading/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzp
 streamlit_octostar_utils/threading/async_task_manager.py,sha256=q7N6YZwUvIYMzkSHmsJNheNVCv93c03H6Hyg9uH8pvk,4747
 streamlit_octostar_utils/threading/session_callback_manager.py,sha256=LvZVP4g6tvKtYmI13f2j1sX_7hm61Groqp5xJine9_k,3973
 streamlit_octostar_utils/threading/session_state_hot_swapper.py,sha256=6eeCQI6A42hp4DmW2NQw2rbeR-k9N8DhfBKQdN_fbLU,811
-streamlit_octostar_utils-2.11a4.dist-info/METADATA,sha256=kRXpmh9YsBrKHNIPYW0gzzASNEwJr8Yj5SJb6gnr4WU,2330
-streamlit_octostar_utils-2.11a4.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-streamlit_octostar_utils-2.11a4.dist-info/licenses/LICENSE,sha256=dkwVPyV03fPHHtERnF6RnvRXcll__tud9gWca1RcgnQ,1073
-streamlit_octostar_utils-2.11a4.dist-info/RECORD,,
+streamlit_octostar_utils-2.11a5.dist-info/METADATA,sha256=sa3ksvvDUHpMWd_szqcaFI_x9u7dVwc9Ctj1gcAyujg,2330
+streamlit_octostar_utils-2.11a5.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+streamlit_octostar_utils-2.11a5.dist-info/licenses/LICENSE,sha256=dkwVPyV03fPHHtERnF6RnvRXcll__tud9gWca1RcgnQ,1073
+streamlit_octostar_utils-2.11a5.dist-info/RECORD,,